JOLT move subarray to parent object and modify contents-CodePudding

I'm trying to move the contents of two subarrays to their parent objects in JOLT to use in NiFi. The Input JSONs are as follows:

{
  "url": "RETURNED URL",
  "repository_url": "RETURNED URL",
  "labels_url": "RETURNED URL",
  "comments_url": "RETURNED URL",
  "events_url": "RETURNED URL",
  "html_url": "RETURNED URL",
  "id": RETURNED_ID,
  "node_id": "RETURNED id",
  "number": 10,
  "title": RETURNED TITLE,
  "user": {
    "login": "xxxx",
    "id": xxxx,
    "node_id": "xxxx",
    "avatar_url": "RETURNED URL",
    "gravatar_id": "",
    "url": "RETURNED URL",
    "html_url": "RETURNED URL",
    "followers_url": "RETURNED URL",
    "following_url": "RETURNED URL",
    "gists_url": "RETURNED URL",
    "starred_url": "RETURNED URL",
    "subscriptions_url": "RETURNED URL",
    "organizations_url": "RETURNED URL",
    "repos_url": "RETURNED URL",
    "events_url": "RETURNED URL",
    "received_events_url": "RETURNED URL",
    "type": "xxxx",
    "site_admin": false
  },
  "labels": [
    {
      "id": 123456,
      "node_id": "RETURNED URL",
      "url": "RETURNED URL",
      "name": xxxxx,
      "color": "53516b",
      "default": false,
      "description": "xxxxx"
    },
    {
      "id": 2345678,
      "node_id": "RETURNED URL",
      "url": "RETURNED URL",
      "name": "xxxxx",
      "color": "B5D1D9",
      "default": false,
      "description": "xxxxxx"
    },
   ...
  ],
  "state": "xxxx",
  "locked": false,
  "assignee": {
    "login": "xxxx",
    "id": 12345,
    "node_id": "RETURNED URL",
    "avatar_url": "RETURNED URL",
    "gravatar_id": "",
    "url": "RETURNED URL",
    "html_url": "RETURNED URL",
    "followers_url": "RETURNED URL",
    "following_url": "RETURNED URL",
    "gists_url": "RETURNED URL",
    "starred_url": "RETURNED URL",
    "subscriptions_url": "RETURNED URL",
    "organizations_url": "RETURNED URL",
    "repos_url": "RETURNED URL",
    "events_url": "RETURNED URL",
    "received_events_url": "RETURNED URL",
    "type": "xxxx",
    "site_admin": false
  },
  "assignees": [
    {
    "login": "xxxx",
    "id": 12345,
    "node_id": "RETURNED URL",
    "avatar_url": "RETURNED URL",
    "gravatar_id": "",
    "url": "RETURNED URL",
    "html_url": "RETURNED URL",
    "followers_url": "RETURNED URL",
    "following_url": "RETURNED URL",
    "gists_url": "RETURNED URL",
    "starred_url": "RETURNED URL",
    "subscriptions_url": "RETURNED URL",
    "organizations_url": "RETURNED URL",
    "repos_url": "RETURNED URL",
    "events_url": "RETURNED URL",
    "received_events_url": "RETURNED URL",
    "type": "xxxx",
    "site_admin": false
  },
    {
    "login": "xxxx",
    "id": 35678,
    "node_id": "RETURNED URL",
    "avatar_url": "RETURNED URL",
    "gravatar_id": "",
    "url": "RETURNED URL",
    "html_url": "RETURNED URL",
    "followers_url": "RETURNED URL",
    "following_url": "RETURNED URL",
    "gists_url": "RETURNED URL",
    "starred_url": "RETURNED URL",
    "subscriptions_url": "RETURNED URL",
    "organizations_url": "RETURNED URL",
    "repos_url": "RETURNED URL",
    "events_url": "RETURNED URL",
    "received_events_url": "RETURNED URL",
    "type": "xxxx",
    "site_admin": false
  },
{
    "login": "xxxx",
    "id": 5785456,
    "node_id": "RETURNED URL",
    "avatar_url": "RETURNED URL",
    "gravatar_id": "",
    "url": "RETURNED URL",
    "html_url": "RETURNED URL",
    "followers_url": "RETURNED URL",
    "following_url": "RETURNED URL",
    "gists_url": "RETURNED URL",
    "starred_url": "RETURNED URL",
    "subscriptions_url": "RETURNED URL",
    "organizations_url": "RETURNED URL",
    "repos_url": "RETURNED URL",
    "events_url": "RETURNED URL",
    "received_events_url": "RETURNED URL",
    "type": "xxxx",
    "site_admin": false
  },
 ...
  ],
  "milestone": null,
  "comments": 1,
  "created_at": "xxxxx",
  "updated_at": "xxxxx",
  "closed_at": null,
  "author_association": "xxxx",
  "active_lock_reason": null,
  "body": "RETURNED_TEXT",
  "performed_via_github_app": null
}

My desired Output:

{
    "id": RETURNED_ID,
    "number": 10,
    "title": RETURNED TITLE,
    "creator": "xxxx",
    "creator_id": "xxxx",
    "creator_admin_stat": false,
    "number_of_comments": 1,
    "created_at": "xxxxx",
    "updated_at": "xxxxx",
    "closed_at": null,
    "issue_body": "xxxx",
    "label_1": {
        "id": 123456
            "name": "xxxxx",
        "description": "xxxxx"
    },
    "label_2": {
        "id": 2345678,
        "name": "xxxxx",
        "description": "xxxxxx"
    },
    // (rest of labels at this same level)
    "user_1": {
        "user_login": "xxxx",
        "user_id": 12345,
        "user_admin_stat": false
    },
    "user_2": {
        "user_login": "xxxx",
        "user_id": 35678,
        "user_admin_stat": false
    },
    "user_3": {
        "user_login": "xxxx",
        "user_id": 578546,
        "user_admin_stat": false
    }
    // (rest of users at this same level)
}

spec I have so far:

[
  {
    "operation": "shift",
    "spec": {
      "id": "repo_id",
      "number": "repo_number",
      "title": "repo_title",
      "user": {
        "login": "creator",
        "id": "creator_id",
        "site_admin": "creator_admin_stat"
      },
      "comments": "number_of_comments",
      "created_at": "created_at",
      "updated_at": "updated_at",
      "closed_at": "closed_at",
      "body": "issue_body",
      "labels": {
        "*": {
          "id": "label_ids",
          "name": "label_names"
        }
      },
      "assignees": {
        "*": {
          "id": "user_ids",
          "login": "usernames"
        }
      }
    }
    }

]

My Current Output:

{
  "repo_id" : 4366853,
  "repo_number" : 10,
  "repo_title" : "xxxx",
  "repo_creator" : "xxxx",
  "repo_creator_id" : 12345,
  "repo_creator_admin_stat" : false,
  "number_of_comments" : 1,
  "created_at" : "xxxx",
  "updated_at" : "xxxx",
  "closed_at" : null,
  "issue_body" : "xxxx",
  "label_ids" : [ // list of label ids],
  "label_names" : [ // list of label names],
  "user_ids" : [ // list of user id numbers ],
  "usernames" : [ // list of usernames ],
  "user_admin_stat" : [ // list of user admin stats ]
}

All of the input JSONs have at least one label, but some might not have any assignees at all. I have a flow built to add null user data if it doesn't exist, so I can put that portion in that flow, but my main objective is trying to get the labels and users into their own sub-objects instead of being in an array.

I'd also like to increment the label/user ids/names like "label_(label count here)" if possible.

These are all going to Kibana and from what I can tell, Kibana doesn't work very well (or at all?) with lists of values, hence my struggle here.

CodePudding user response：

You can use the following spec, except for some missing or non-conforming key-value pairs, and indexing the arrays labels and assignees starting with zero

[
  {
    "operation": "shift",
    "spec": {
      "id": "&",
      "number": "&",
      "title": "&",
      "created_at": "&",
      "labels": {
        "*": {
          "id": "&2_&1.&",
          "name": "&2_&1.&",
          "description": "&2_&1.&"
        }
      },
      "assignees": {
        "*": {
          "login": "user_&1.user_&",
          "id": "user_&1.user_&",
          "site_admin": "user_&1.user_admin_stat"
        }
      }
    }
  }
]

yielding output

{
  "id" : "RETURNED_ID",
  "number" : 10,
  "title" : "RETURNED TITLE",
  "created_at" : "xxxxx",
  "labels_0" : {
    "id" : 123456,
    "name" : "xxxxx",
    "description" : "xxxxx"
  },
  "labels_1" : {
    "id" : 2345678,
    "name" : "xxxxx",
    "description" : "xxxxxx"
  },
  "user_0" : {
    "user_login" : "xxxx",
    "user_id" : 12345,
    "user_admin_stat" : false
  },
  "user_1" : {
    "user_login" : "xxxx",
    "user_id" : 35678,
    "user_admin_stat" : false
  },
  "user_2" : {
    "user_login" : "xxxx",
    "user_id" : 5785456,
    "user_admin_stat" : false
  }
}