I'm trying to transfer camera stream from my raspberry pi to pc as fast as possible. Now I'm trying to transfer it using MJPEG method. So I have a python script on my raspberry as server:
import io
import socket
import struct
import time
import cv2
class SplitFrames(object):
def __init__(self, connection):
self.connection = connection
self.stream = io.BytesIO()
self.count = 0
def write(self, buf):
if buf.startswith(b'\xff\xd8'):
# Start of new frame; send the old one's length
# then the data
size = self.stream.tell()
if size > 0:
self.connection.write(struct.pack('<L', size))
self.connection.flush()
self.stream.seek(0)
self.connection.write(self.stream.read(size))
self.count = 1
self.stream.seek(0)
self.stream.write(buf)
server_socket = socket.socket()
server_socket.bind(('0.0.0.0', 8001))
server_socket.listen(0)
# Accept a single connection and make a file-like object out of it
connection = server_socket.accept()[0].makefile('wb')
try:
output = SplitFrames(connection)
time.sleep(2)
cap = cv2.VideoCapture(0)
while True:
ret, img = cap.read()
ret, jpg = cv2.imencode('.jpg', img)
output.write(jpg.tostring())
connection.write(struct.pack('<L', 0))
finally:
connection.close()
server_socket.close()
that works well. Also I have a python script on my pc as client, that also works well and I receive a good stream:
import io
import socket
import struct
from PIL import Image
import cv2
import numpy as np
# Start a socket listening for connections on 0.0.0.0:8000 (0.0.0.0 means
# all interfaces)
client_socket = socket.socket()
client_socket.connect(("10.12.34.2", 8001))
connection = client_socket.makefile('rb')
try:
while True:
# Read the length of the image as a 32-bit unsigned int. If the
# length is zero, quit the loop
a = connection.read(struct.calcsize('<L'))
image_len = struct.unpack('<L', a)[0]
if not image_len:
break
# Construct a stream to hold the image data and read the image
# data from the connection
image_stream = io.BytesIO()
image_stream.write(connection.read(image_len))
# Rewind the stream, open it as an image with PIL and do some
# processing on it
image_stream.seek(0)
image = Image.open(image_stream)
cv_image = np.array(image)
cv2.imshow('Stream',cv_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
finally:
connection.close()
client_socket.close()
But I needed to write client side on WPF, so I wrote it, but it doesn't work well. I receive distorted stream and after a few seconds WPF app breaks and gives me System.IO.FileFormatException. Here is the image from WPF:
What could be the reason of this? Thanks for any advice!
Here is the minimum reproducible code:
C#
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Windows;
using System.Windows.Media.Imaging;
namespace WPFTest
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
private static bool connected = false;
private Socket sct;
private IPEndPoint ipPoint;
// TCP
private bool stopThread = false;
private Thread thread;
public MainWindow()
{
InitializeComponent();
}
private void ConnectButton_Click(object sender, RoutedEventArgs e)
{
if (!connected)
{
try
{
//this.ipPoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), int.Parse("8001", CultureInfo.InvariantCulture));
this.ipPoint = new IPEndPoint(IPAddress.Parse("10.12.34.2"), int.Parse("8001", CultureInfo.InvariantCulture));
}
catch (Exception error)
{
return;
}
this.sct = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP);
stopThread = false;
thread = new Thread(() => { Listening(); });
thread.IsBackground = false;
thread.Start();
connected = true;
}
}
private void DisconnectButton_Click(object sender, RoutedEventArgs e)
{
if (connected)
{
StopListening();
connected = false;
}
}
private void Listening()
{
try
{
sct.Connect(ipPoint);
}
catch (Exception e)
{
StopListening();
connected = false;
return;
}
Thread.Sleep(1000);
try
{
while (!this.stopThread)
{
byte[] data = GetInputBytes(sct);
Application.Current.Dispatcher.Invoke(() =>
{
SetImage(data);
});
}
sct.Shutdown(SocketShutdown.Both);
sct.Close();
}
catch (Exception e)
{
StopListening();
}
}
public void StopListening()
{
this.stopThread = true;
try
{
thread.Abort();
sct.Shutdown(SocketShutdown.Both);
sct.Close();
}
catch (Exception e)
{
//
}
}
private void SetImage(byte[] array)
{
var image = new BitmapImage();
using (var mem = new MemoryStream(array))
{
mem.Position = 0;
image.BeginInit();
image.CreateOptions = BitmapCreateOptions.PreservePixelFormat;
image.CacheOption = BitmapCacheOption.OnLoad;
image.UriSource = null;
image.StreamSource = mem;
image.EndInit();
}
image.Freeze();
ImageCamera.Source = image;
}
public static byte[] GetInputBytes(Socket clientSocket)
{
byte[] rcvLenBytes = new byte[4];
clientSocket.Receive(rcvLenBytes);
UInt32 rcvLen = BytesToInt(rcvLenBytes);
byte[] rcvBytes;
byte[] clientData;
List<byte> rcvBytesList = new List<byte>();
int totalBytes = 0;
while (totalBytes < rcvLen)
{
if (rcvLen - totalBytes < 262144)
{
clientData = new byte[rcvLen - totalBytes];
}
else
{
clientData = new byte[262144];
}
int bytesReceived = clientSocket.Receive(clientData);
rcvBytesList.AddRange(clientData);
totalBytes = bytesReceived;
}
rcvBytes = rcvBytesList.ToArray();
return rcvBytes;
}
public static UInt32 BytesToInt(byte[] arr)
{
UInt32 wd = ((UInt32)arr[3] << 24) | ((UInt32)arr[2] << 16) | ((UInt32)arr[1] << 8) | (UInt32)arr[0];
return wd;
}
}
}
XAML
<Window x:Class="WPFTest.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:local="clr-namespace:WPFTest"
mc:Ignorable="d"
Title="MainWindow" Height="450" Width="800">
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="*"></RowDefinition>
<RowDefinition Height="*"></RowDefinition>
<RowDefinition Height="200px"></RowDefinition>
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*" ></ColumnDefinition>
<ColumnDefinition Width="220px" ></ColumnDefinition>
<ColumnDefinition Width="*" ></ColumnDefinition>
<ColumnDefinition Width="180px" ></ColumnDefinition>
</Grid.ColumnDefinitions>
<Button Grid.Column="2" Grid.Row="3" Height="60" Width="140" Content="Connect" x:Name="ConnectButton" FontSize="20" FontFamily="LilyUPC" FontWeight="Bold" HorizontalAlignment="Left" VerticalAlignment="Top" Margin="20, 80, 20, 0" Click="ConnectButton_Click"/>
<Button Grid.Column="0" Grid.Row="3" Height="60" Width="140" Content="Disconnect" x:Name="DisconnectButton" FontSize="20" FontFamily="LilyUPC" FontWeight="Bold" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="20, 80, 20, 0" Click="DisconnectButton_Click"/>
<Image Grid.Column="0" Grid.Row="0" Grid.RowSpan="2" Grid.ColumnSpan="3" Margin="10" RenderOptions.BitmapScalingMode="NearestNeighbor" RenderOptions.EdgeMode="Aliased" x:Name="ImageCamera"></Image>
</Grid>
</Window>
CodePudding user response：
Thanks to @ThomasWeller for their answer in comments! As they said:
AddRange adds the whole buffer. How should it know how many bytes were received?
So I rewrote my receive method like this:
public static byte[] GetInputBytes(Socket clientSocket)
{
byte[] rcvLenBytes = new byte[4];
clientSocket.Receive(rcvLenBytes);
UInt32 rcvLen = BytesToInt(rcvLenBytes);
byte[] rcvBytes;
byte[] clientData;
List<byte> rcvBytesList = new List<byte>();
int totalBytes = 0;
while (totalBytes < rcvLen)
{
if (rcvLen - totalBytes < 262144)
{
clientData = new byte[rcvLen - totalBytes];
}
else
{
clientData = new byte[262144];
}
int bytesReceived = clientSocket.Receive(clientData);
rcvBytesList.AddRange(clientData.Take(bytesReceived).ToArray());
totalBytes = bytesReceived;
}
rcvBytes = rcvBytesList.ToArray();
return rcvBytes;
}
Changed line is: rcvBytesList.AddRange(clientData.Take(bytesReceived).ToArray());
It works fine now.