Home > Software design >  protobuf decode unknown message structure
protobuf decode unknown message structure

Time:12-30

Scenario

For a map tiling project (think google earth) I have a .mbtile file that I know nothing about other than it has a lot of data about cities on the planet. I've been picking at it for a couple days and have found out the following:

  • the file is a sqlite database
  • the database has a table tiles which is full of blobs
  • the blobs are binary with gzip signatures
  • after unzipping, the result is a protocol buffer

I haven't worked with protobuf much so I'm getting a crash course. I took the unzipped binary and with protoc --decode_raw received the following

$ echo 1a900178010a06706c6163657328802012121208000001010202030318012204095cd6341213120800040105020603071801220509e206b2361a047a6d696e1a046e616d651a0a706f70756c6174696f6e1a0773756274696c652202280322070a053f73616b61220528b0aab105220328c006220919333333333333fb3f22070a05546f6b796f220528e0be8111220328e306 | xxd -r -p | protoc --decode_raw
3 {
  15: 1
  1: "places"
  5: 4096
  2 {
    2: "\000\000\001\001\002\002\003\003"
    3: 1
    4: "\t\\\3264"
  }
  2 {
    2: "\000\004\001\005\002\006\003\007"
    3: 1
    4: "\t\342\006\2626"
  }
  3: "zmin"
  3: "name"
  3: "population"
  3: "subtile"
  4 {
    5: 3
  }
  4 {
    1: "?saka"
  }
  4 {
    5: 11294000
  }
  4 {
    5: 832
  }
  4 {
    3: 0x3ffb333333333333
  }
  4 {
    1: "Tokyo"
  }
  4 {
    5: 35676000
  }
  4 {
    5: 867
  }
}

Question

If I understand correctly, this tells me a bit about the message structure. How can I go about creating a [ProtoContract] for this output? Below is my current (unworking) attempt

[ProtoContract]
struct Place
{
    [ProtoMember(1)]
    public string name { get; set; }

    [ProtoMember(2)]
    public double population { get; set; }

    [ProtoMember(3)]
    public uint subtile { get; set; }

    [ProtoMember(4)]
    public double zmin { get; set; }

    [ProtoMember(5)]
    public int[] location { get; set; }
}

** SOLUTION **

working c# protobuf contracts thx to @MarcGravell

#region Designer generated code
#pragma warning disable CS0612, CS0618, CS1591, CS3021, IDE0079, IDE1006, RCS1036, RCS1057, RCS1085, RCS1192
namespace MapBox
{

    [global::ProtoBuf.ProtoContract()]
    public partial class Tile : global::ProtoBuf.IExtensible
    {
        private global::ProtoBuf.IExtension __pbn__extensionData;
        global::ProtoBuf.IExtension global::ProtoBuf.IExtensible.GetExtensionObject(bool createIfMissing)
            => global::ProtoBuf.Extensible.GetExtensionObject(ref __pbn__extensionData, createIfMissing);

        [global::ProtoBuf.ProtoMember(3, Name = @"layers")]
        public global::System.Collections.Generic.List<Layer> Layers { get; } = new global::System.Collections.Generic.List<Layer>();

        [global::ProtoBuf.ProtoContract()]
        public partial class Value : global::ProtoBuf.IExtensible
        {
            private global::ProtoBuf.IExtension __pbn__extensionData;
            global::ProtoBuf.IExtension global::ProtoBuf.IExtensible.GetExtensionObject(bool createIfMissing)
                => global::ProtoBuf.Extensible.GetExtensionObject(ref __pbn__extensionData, createIfMissing);

            [global::ProtoBuf.ProtoMember(1, Name = @"string_value")]
            [global::System.ComponentModel.DefaultValue("")]
            public string StringValue
            {
                get => __pbn__StringValue ?? "";
                set => __pbn__StringValue = value;
            }
            public bool ShouldSerializeStringValue() => __pbn__StringValue != null;
            public void ResetStringValue() => __pbn__StringValue = null;
            private string __pbn__StringValue;

            [global::ProtoBuf.ProtoMember(2, Name = @"float_value")]
            public float FloatValue
            {
                get => __pbn__FloatValue.GetValueOrDefault();
                set => __pbn__FloatValue = value;
            }
            public bool ShouldSerializeFloatValue() => __pbn__FloatValue != null;
            public void ResetFloatValue() => __pbn__FloatValue = null;
            private float? __pbn__FloatValue;

            [global::ProtoBuf.ProtoMember(3, Name = @"double_value")]
            public double DoubleValue
            {
                get => __pbn__DoubleValue.GetValueOrDefault();
                set => __pbn__DoubleValue = value;
            }
            public bool ShouldSerializeDoubleValue() => __pbn__DoubleValue != null;
            public void ResetDoubleValue() => __pbn__DoubleValue = null;
            private double? __pbn__DoubleValue;

            [global::ProtoBuf.ProtoMember(4, Name = @"int_value")]
            public long IntValue
            {
                get => __pbn__IntValue.GetValueOrDefault();
                set => __pbn__IntValue = value;
            }
            public bool ShouldSerializeIntValue() => __pbn__IntValue != null;
            public void ResetIntValue() => __pbn__IntValue = null;
            private long? __pbn__IntValue;

            [global::ProtoBuf.ProtoMember(5, Name = @"uint_value")]
            public ulong UintValue
            {
                get => __pbn__UintValue.GetValueOrDefault();
                set => __pbn__UintValue = value;
            }
            public bool ShouldSerializeUintValue() => __pbn__UintValue != null;
            public void ResetUintValue() => __pbn__UintValue = null;
            private ulong? __pbn__UintValue;

            [global::ProtoBuf.ProtoMember(6, Name = @"sint_value", DataFormat = global::ProtoBuf.DataFormat.ZigZag)]
            public long SintValue
            {
                get => __pbn__SintValue.GetValueOrDefault();
                set => __pbn__SintValue = value;
            }
            public bool ShouldSerializeSintValue() => __pbn__SintValue != null;
            public void ResetSintValue() => __pbn__SintValue = null;
            private long? __pbn__SintValue;

            [global::ProtoBuf.ProtoMember(7, Name = @"bool_value")]
            public bool BoolValue
            {
                get => __pbn__BoolValue.GetValueOrDefault();
                set => __pbn__BoolValue = value;
            }
            public bool ShouldSerializeBoolValue() => __pbn__BoolValue != null;
            public void ResetBoolValue() => __pbn__BoolValue = null;
            private bool? __pbn__BoolValue;

        }

        [global::ProtoBuf.ProtoContract()]
        public partial class Feature : global::ProtoBuf.IExtensible
        {
            private global::ProtoBuf.IExtension __pbn__extensionData;
            global::ProtoBuf.IExtension global::ProtoBuf.IExtensible.GetExtensionObject(bool createIfMissing)
                => global::ProtoBuf.Extensible.GetExtensionObject(ref __pbn__extensionData, createIfMissing);

            [global::ProtoBuf.ProtoMember(1, Name = @"id")]
            [global::System.ComponentModel.DefaultValue(typeof(ulong), "0")]
            public ulong Id
            {
                get => __pbn__Id ?? 0;
                set => __pbn__Id = value;
            }
            public bool ShouldSerializeId() => __pbn__Id != null;
            public void ResetId() => __pbn__Id = null;
            private ulong? __pbn__Id;

            [global::ProtoBuf.ProtoMember(2, Name = @"tags", IsPacked = true)]
            public uint[] Tags { get; set; }

            [global::ProtoBuf.ProtoMember(3, Name = @"type")]
            [global::System.ComponentModel.DefaultValue(Tile.GeomType.Unknown)]
            public Tile.GeomType Type
            {
                get => __pbn__Type ?? Tile.GeomType.Unknown;
                set => __pbn__Type = value;
            }
            public bool ShouldSerializeType() => __pbn__Type != null;
            public void ResetType() => __pbn__Type = null;
            private Tile.GeomType? __pbn__Type;

            [global::ProtoBuf.ProtoMember(4, Name = @"geometry", IsPacked = true)]
            public uint[] Geometries { get; set; }

        }

        [global::ProtoBuf.ProtoContract()]
        public partial class Layer : global::ProtoBuf.IExtensible
        {
            private global::ProtoBuf.IExtension __pbn__extensionData;
            global::ProtoBuf.IExtension global::ProtoBuf.IExtensible.GetExtensionObject(bool createIfMissing)
                => global::ProtoBuf.Extensible.GetExtensionObject(ref __pbn__extensionData, createIfMissing);

            [global::ProtoBuf.ProtoMember(15, Name = @"version", IsRequired = true)]
            public uint Version { get; set; }

            [global::ProtoBuf.ProtoMember(1, Name = @"name", IsRequired = true)]
            public string Name { get; set; }

            [global::ProtoBuf.ProtoMember(2, Name = @"features")]
            public global::System.Collections.Generic.List<Tile.Feature> Features { get; } = new global::System.Collections.Generic.List<Tile.Feature>();

            [global::ProtoBuf.ProtoMember(3, Name = @"keys")]
            public global::System.Collections.Generic.List<string> Keys { get; } = new global::System.Collections.Generic.List<string>();

            [global::ProtoBuf.ProtoMember(4, Name = @"values")]
            public global::System.Collections.Generic.List<Tile.Value> Values { get; } = new global::System.Collections.Generic.List<Tile.Value>();

            [global::ProtoBuf.ProtoMember(5, Name = @"extent")]
            [global::System.ComponentModel.DefaultValue(4096)]
            public uint Extent
            {
                get => __pbn__Extent ?? 4096;
                set => __pbn__Extent = value;
            }
            public bool ShouldSerializeExtent() => __pbn__Extent != null;
            public void ResetExtent() => __pbn__Extent = null;
            private uint? __pbn__Extent;

        }

        [global::ProtoBuf.ProtoContract()]
        public enum GeomType
        {
            [global::ProtoBuf.ProtoEnum(Name = @"UNKNOWN")]
            Unknown = 0,
            [global::ProtoBuf.ProtoEnum(Name = @"POINT")]
            Point = 1,
            [global::ProtoBuf.ProtoEnum(Name = @"LINESTRING")]
            Linestring = 2,
            [global::ProtoBuf.ProtoEnum(Name = @"POLYGON")]
            Polygon = 3,
        }

    }

}

#pragma warning restore CS0612, CS0618, CS1591, CS3021, IDE0079, IDE1006, RCS1036, RCS1057, RCS1085, RCS1192
#endregion

CodePudding user response:

Try the Mapbox Vector Tile Specification.

  • Related