I am working on my online file storage and today I have encountered some issues with my Netty TCP file tranfer. So the problem is that only 8192 bytes of data is actually written in the file on the client-side. I want to know what the problem is, and how I can fix it.
I have seen ALL of the other (5) stackoverflow questions.
Here is my server bootstrap:
package com.martin.main;
import com.martin.file.*;
import com.martin.handler.*;
import com.martin.utils.*;
import io.netty.bootstrap.*;
import io.netty.channel.*;
import io.netty.channel.nio.*;
import io.netty.channel.socket.*;
import io.netty.channel.socket.nio.*;
import io.netty.handler.codec.*;
import java.io.*;
import java.net.*;
import java.util.*;
import java.util.concurrent.*;
public class Server {
public static void main(String[] args) {
new Server().setup(4783);
}
public ChannelFuture setup(int port) {
ChannelFuture future = null;
EventLoopGroup bossGroup = new NioEventLoopGroup();
EventLoopGroup workerGroup = new NioEventLoopGroup();
final ServerBootstrap bootstrap;
// OnlineFile.getOnlineFiles().add(new OnlineFile(737389, new File("C:\\Users\\marti\\Desktop\\filesharing\\testek.txt"), "ok.txt", (int) System.currentTimeMillis(), false, "C:\\Users\\marti\\Desktop\\filesharing\\testek.txt", false, null));
try {
bootstrap = new ServerBootstrap();
bootstrap.group(bossGroup, workerGroup);
bootstrap.channel(NioServerSocketChannel.class);
bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
ChannelPipeline pipeline = socketChannel.pipeline();
//pipeline.addLast("framer", new LengthFieldBasedFrameDecoder());
pipeline.addLast("framer", new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4));
pipeline.addLast(new LengthFieldPrepender(4));
pipeline.addLast("login", new LoginServerHandler()); //the problem is not in that handler
}
});
future = bootstrap.bind(new InetSocketAddress(port)).sync().channel().closeFuture().sync();
}catch(InterruptedException e) {
e.printStackTrace();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
return future;
}
}
My server-handler:
public class ServerHandler extends ByteToMessageDecoder {
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
cause.printStackTrace();
}
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List<Object> list) throws Exception {
byte msgType = byteBuf.readByte();
if(FILEPACKETINFO == msgType) {
//The problem is probaly here
System.out.println("inide file req.");
int id = byteBuf.readInt();
for(OnlineFile onlineFile : OnlineFile.getOnlineFiles()) {
if(onlineFile.getId() == id) {
System.out.println("file request id: " id " actual id: " onlineFile.getId());
ByteBuf buf = Unpooled.buffer();
buf.writeByte(FILEPACKETINFO);
String fileName = onlineFile.getName();
File file = onlineFile.getActualFile();
buf.writeLong(file.length());
buf.writeInt(fileName.length());
buf.writeCharSequence(fileName, CharsetUtil.US_ASCII);
ctx.writeAndFlush(buf);
if(!(file.length() <= 0)) {
ctx.writeAndFlush(new ChunkedFile(file)).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture channelFuture) throws Exception {
System.out.println("written successfully!");
if (channelFuture.cause() != null) channelFuture.cause().printStackTrace();
}
});
} else {
System.out.println("length 0: " " (" file.length() ")");
}
}
}
} else if(REQUESTVISUALFILES == msgType) {
createAndSend(ctx); //send VISUAL files, problem not there
} else if(REQUESTFOLDERFILESBYID == msgType) {
int id = byteBuf.readInt();
createAndSend(ctx, id); //send VISUAL files by id, problem not there
}
}
My client bootstrap:
public static ChannelFuture setup(String host, int port) {
ChannelFuture channelFuture = null;
EventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(group);
bootstrap.channel(NioSocketChannel.class);
bootstrap.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
ChannelPipeline pipeline = socketChannel.pipeline();
pipeline.addLast("framer", new LengthFieldBasedFrameDecoder(64*1024, 0, 4, 0, 4));
pipeline.addLast(lfp);
pipeline.addLast("login", new LoginHandler()); //the problem is not in that handler either
}
});
channelFuture = bootstrap.connect(new InetSocketAddress(host, port)).sync();
System.out.println("the setup came to an end!");
LoginForm.createAndShowGUI("Login");
mainChannel = channelFuture.channel();
channelFuture.channel().closeFuture().sync();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
group.shutdownGracefully();
}
return channelFuture;
}
}
My client handler:
package com.martin.handler;
import io.netty.buffer.*;
import io.netty.channel.*;
import io.netty.util.*;
import java.io.*;
import java.nio.*;
import java.nio.channels.*;
public class FileChunkHandler extends SimpleChannelInboundHandler<ByteBuf> {
public static String currentFileName = "Test.txt";
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
cause.printStackTrace();
ctx.pipeline().remove(this);
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception {
//the problem is probably here
System.out.println("inside FileChunkHandler current file: " currentFileName);
ByteBuffer buffer = byteBuf.nioBuffer();
System.out.println(buffer.capacity() " buffer" " bytebuf: " byteBuf.readableBytes());
File file = triggerFileCreation();
// FileOutputStream fos = new FileOutputStream("C:\\Users\\marti\\storage\\" currentFileName);
RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw");
FileChannel channel = randomAccessFile.getChannel();
while(buffer.hasRemaining()) {
channel.position(file.length());
channel.write(buffer);
System.out.println("chunk has just been written");
}
channel.close();
randomAccessFile.close();
//!!!--->IMPORTANT<-----!!!
ctx.pipeline().remove(this);
}
public static File triggerFileCreation() {
File file = new File(System.getProperty("user.home") "/storage/" currentFileName);
if(!file.exists()) {
try {
file.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
}
return file;
}
}
Thanks for help!
CodePudding user response：
The problem is indeed most likely to be in your FileChunkHandler.
You only read a single buffer (likely containing 8192 bytes - 8kB), and then remove the handler. The remaining chunks either get "handled" by some other handler in the pipeline, or reach the end of the pipeline and get dropped. As mentioned in Discord, you need to keep track of how many bytes you are expecting, subtract the number received, and only when that number reaches 0, should you remove the handler.
CodePudding user response：
Some code optimizations can be done here
If all you are trying to do is to transfer data from your ByteBuffer to an file then Replace this code
@Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception {
//the problem is probably here
System.out.println("inside FileChunkHandler current file: " currentFileName);
ByteBuffer buffer = byteBuf.nioBuffer();
System.out.println(buffer.capacity() " buffer" " bytebuf: " byteBuf.readableBytes());
File file = triggerFileCreation();
// FileOutputStream fos = new FileOutputStream("C:\\Users\\marti\\storage\\" currentFileName);
RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw");
FileChannel channel = randomAccessFile.getChannel();
while(buffer.hasRemaining()) {
channel.position(file.length());
channel.write(buffer);
System.out.println("chunk has just been written");
}
channel.close();
randomAccessFile.close();
//!!!--->IMPORTANT<-----!!!
ctx.pipeline().remove(this);
}
public static File triggerFileCreation() {
File file = new File(System.getProperty("user.home") "/storage/" currentFileName);
if(!file.exists()) {
try {
file.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
}
return file;
}
With this
@Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception
{
System.out.println("inside FileChunkHandler current file: " currentFileName);
ByteBuffer buffer = byteBuf.nioBuffer();
System.out.println(buffer.capacity() " buffer" " bytebuf: " byteBuf.readableBytes());
try(FileChannel channel=FileChannel.open(Path.of(System.getProperty("user.home"),"/storage/",currentFileName),StandardOpenOption.CREATE,StandardOpenOption.APPEND))
{
while(buffer.hasRemaining())
{
channel.write(buffer);
System.out.println("chunk has just been written");
}
}
//!!!--->IMPORTANT<-----!!!
ctx.pipeline().remove(this);
}
As to why you only get 8KB of bytes from the server i am unsure. Maybe your server reuses the same ByteBuffer and if that's the case then you have to clear it to make its position=0 and limit=buffer.capacity() after writing it to your file so the server can use the new space to write more bytes to it
And also since you are register an handler to receive new chunks it maybe an good idea to remove the handler after you file has reached the expected size
Putting these two together the final code is as follows
@Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception
{
System.out.println("inside FileChunkHandler current file: " currentFileName);
ByteBuffer buffer = byteBuf.nioBuffer();
System.out.println(buffer.capacity() " buffer" " bytebuf: " byteBuf.readableBytes());
try(FileChannel channel=FileChannel.open(Path.of(System.getProperty("user.home"),"/storage/",currentFileName),StandardOpenOption.CREATE,StandardOpenOption.APPEND))
{
while(buffer.hasRemaining())
{
channel.write(buffer);
System.out.println("chunk has just been written");
}
//clear the buffer so the server has space to transfer new chunks into it
buffer.clear();
//Remove handler only after file has reached an certain size
if(channel.size()>=expectedLength){
ctx.pipeline().remove(this);
}
}
}
Apart from the code optimization the other's are just guesses and may not be the answer you were looking for. Comment below if any progress was made