Group multiple barcodes on a single sticker together from an image with multiple stickers using pyth-CodePudding

Objective: Automate the scanning of our product's barcodes into our shipping program using the Python language.

Situation: Each sticker on a product has two barcodes. One (the SKU) identifies what the product line is, and the other (serial number) is a unique ID identifying it from the others in the same product line. For example, in an image, there could be ten stickers with the same SKU of, say, "Product A" and all ten of those stickers have unique serial numbers. There could also be "Product B" and "Product C" in the image as well.

Progress: I can use pyzbar and cv2 to scan multiple barcodes in an image successfully.

Issue: I want to group the SKU and Serial number barcodes by sticker, but I don't know how to do this or where to start.

Code I am using

from pyzbar.pyzbar import decode, ZBarSymbol
import cv2

testing_image_readin = cv2.imread(testing_image_path)
detected_barcodes = decode(testing_image_readin, symbols=[ZBarSymbol.CODE128, ZBarSymbol.EAN13])

if not detected_barcodes:
    print("Barcode Not Detected or your barcode is blank/corrupted!")
else:
    for barcode in detected_barcodes:
        # Locate the barcode position in image
        (x, y, w, h) = barcode.rect

        cv2.rectangle(testing_image_readin, (x - 10, y - 10),
                      (x   w   10, y   h   10),
                      (255, 0, 0), 2)

        if barcode.data != "":
            # Print the barcode data
            print(barcode.data)
            print(barcode.type)

UPDATE - Adding Example Images:

I dont have an example of the exact image I am describing so I have made one in with graphics. This would be a top-down image looking at the stickers on the Product Boxes.

Example Box:

Program output:

b'07FFD58D47189877'
CODE128
b'0871828002084'
EAN13

Generated Top Down view of multiple boxes together All with unique serial numbers:

CodePudding user response：

Ok since pyzbar/zbar seems to have bugs that cause its bounding boxes to catch multiple codes, or not detect codes that are rotated too much, I'll use OpenCV's barcode detection, rectify the codes, then use pyzbar for decoding. OpenCV can also decode, but not as many different types.

Approach:

find codes, get bounding boxes
for each bounding box, enlarge it width-wise and see what other codes it overlaps with
build "groups"

Input:

Detect barcodes with OpenCV:

det = cv.barcode.BarcodeDetector()
(rv, detections) = det.detect(im)
# detections: four corner points per detection

Extract rectangle:

def extract_region_from_corners(image, corners):
    # order:
    #   [1] top left     [2] top right
    #   [0] bottom left  [3] bottom right
    (bl, tl, tr, br) = corners

    # axis vectors
    vx = tr - tl
    vy = bl - tl

    lx = np.linalg.norm(vx)
    ly = np.linalg.norm(vy)

    H = np.eye(3)
    H[0:2,2] = tl # origin
    H[:2,0] = vx / lx
    H[:2,1] = vy / ly

    dst = cv.warpAffine(src=image,
        M=H[:2], dsize=(int(lx), int(ly)),
        flags=cv.INTER_LINEAR | cv.WARP_INVERSE_MAP)

    return dst

Utility function:

def corners_to_rrect(corners):
    # order:
    #   [1] top left     [2] top right
    #   [0] bottom left  [3] bottom right
    (bl, tl, tr, br) = corners

    vx = ((tr - tl)   (br - bl)) / 2
    vy = ((bl - tl)   (br - tr)) / 2
    lx = np.linalg.norm(vx)
    ly = np.linalg.norm(vy)

    center = tuple(corners.mean(axis=0))
    size = (lx, ly)
    angle = np.arctan2(vx[1], vx[0]) / np.pi * 180 # degrees

    return (center, size, angle)

Extract codes, decode, note their RotatedRect positions:

found_codes = []
canvas = im.copy()

for detection_corners in detections:
    rrect = corners_to_rrect(detection_corners)

    (rrect_width, rrect_height) = rrect[1]
    assert rrect_width > rrect_height, ("assuming barcode lies lengthwise", rrect)

    roi = extract_region_from_corners(image=im, corners=detection_corners)

    [code] = pyzbar.decode(roi, symbols=[ZBarSymbol.CODE128, ZBarSymbol.EAN13])
    print(code.type, code.data, rrect)

    found_codes.append( (rrect, code) )

    cv.polylines(img=canvas, pts=[detection_corners.astype(np.int32)], isClosed=True, color=(255, 0, 0), thickness=2)

CODE128 b'07FFD58D47189879' ((706.9937, 355.28094), (434.7604, 65.09412), 15.141749040805594)
CODE128 b'07FFD58D47189878' ((266.48895, 361.89154), (435.78812, 65.95062), -15.051276355059604)
CODE128 b'07FFD58D47189876' ((237.65492, 816.5005), (434.7883, 65.28357), 15.058296081979087)
CODE128 b'07FFD58D47189877' ((731.69257, 817.5774), (435.56052, 62.905884), -15.084296904602034)
EAN13 b'0871828002084' ((228.3433, 239.54503), (235.90378, 66.31835), -15.219580753945182)
EAN13 b'0871828002077' ((705.7166, 693.0964), (236.39447, 65.9507), -15.102472037983436)
EAN13 b'0871828002091' ((742.64703, 237.18982), (240.23358, 67.790794), 15.171352788215723)
EAN13 b'0871828002060' ((270.11478, 696.054), (236.27463, 64.16398), 15.201185346963047)

More utility functions:

def enlarge_rrect(rrect, factor=1, fx=1, fy=1):
    (center, size, angle) = rrect
    (width, height) = size
    new_size = (width * factor * fx, height * factor * fy)
    return (center, new_size, angle)


def merge_intersecting_sets(sets):
    # sets = set(map(frozenset, sets))
    while True:
        oldcount = len(sets)

        # merge or add
        newsets = set()
        for thisset in sets:
            for thatset in newsets:
                if thisset & thatset:
                    newsets.remove(thatset)
                    newsets.add(thisset | thatset)
                    break
            else:
                newsets.add(thisset)

        sets = newsets
        if len(sets) == oldcount:
            break
    
    return sets

# assert merge_intersecting_sets([{1,2}, {2,3}, {3,4}, {5,6}]) == {frozenset({1,2,3,4}), frozenset({5,6})}

Determine groups using enlarged RotatedRect and intersection test:

def associate_rrects(rrects, fx=1, fy=1):
    "associate RotatedRect instances, given enlargement factors in horizontal and vertical direction"

    # build connected components by adjacency
    components = set()
    for (i, thisrect) in enumerate(rrects):
        thisenlarged = enlarge_rrect(thisrect, fx=fx, fy=fy)

        component = {i}
        for (j, thatrect) in enumerate(rrects):
            (rv, intersection) = cv.rotatedRectangleIntersection(thisenlarged, thatrect)
            if rv != cv.INTERSECT_NONE: # i.e. INTERSECT_PARTIAL, INTERSECT_FULL
                component.add(j)

        components.add(frozenset(component))
    
    # merge intersecting components (transitivitiy)
    components = merge_intersecting_sets(components)

    return components


components = associate_rrects([rrect for rrect, code in found_codes], fy=5)
print(components)

{frozenset({1, 4}), frozenset({2, 7}), frozenset({0, 6}), frozenset({3, 5})}

Now you can pick from found_codes using those indices.

Drawing the groups, using convex hull:

canvas = im.copy()
for component in components:
    component_codes = [found_codes[i] for i in component]

    component_corners = np.concatenate([
        cv.boxPoints(rrect)
        for (rrect, code) in component_codes
    ])
    hull = cv.convexHull(component_corners)
    cv.polylines(img=canvas, pts=[hull.astype(np.int32)], isClosed=True, color=(255, 255, 0), thickness=2)

    for (rrect, code) in component_codes:
        #print(rrect, code)
        cv.polylines(img=canvas, pts=[cv.boxPoints(rrect).astype(int)], isClosed=True, color=(255, 0, 0), thickness=2)
        cv.putText(canvas, text=str(code.data),
            org=np.int0(rrect[0]), fontFace=cv.FONT_HERSHEY_SIMPLEX,
            fontScale=0.7, color=(0,0,0), thickness=8)
        cv.putText(canvas, text=str(code.data),
            org=np.int0(rrect[0]), fontFace=cv.FONT_HERSHEY_SIMPLEX,
            fontScale=0.7, color=(0,255,255), thickness=2)

Entire thing: https://gist.github.com/crackwitz/3a7e7e5d698274198393737415ef409a