Being new to python, I am unable to resolve the following issue.

Below is my python code, which returns different json outputs compared to when executed with list passed in a variable vs each line read from a file.

Code with lines read from file which throws partial/corrupted output:

import requests
import json
import re

repo_name = "repo"

with open('file_list_new.txt') as file:
    for line in file:
       url = "http://fqdn/repository/{0}/{1}?describe=json".format(repo_name, line)
       response = requests.get(url)
       json_data = response.text
       data = json.loads(json_data)
       print(data)
       for size in data['items']:
          if size['name'] == 'Payload':
            value_size= size['value']['Size']
            if value_size != -1:
              print(value_size)

content of file_list_new.txt

mysql.odbc/5.1.14
mysql.odbc/5.1.11

corrupted output

{
  "parameters" : {
    "path" : "/mysql.odbc/5.1.14\n",
    "nexusUrl" : "http://fqdn"
  },
  "items" : [ {
    "name" : "Exception during handler processing",
    "type" : "topic",
    "value" : "Exception during handler processing"
  }, {
    "name" : "java.lang.IllegalArgumentException",
    "type" : "table",
    "value" : {
      "Message" : "Illegal character in path at index 40: Packages(Id='mysql.odbc',Version='5.1.14\n')"
    }
  }, {
    "name" : "java.net.URISyntaxException",
    "type" : "table",
    "value" : {
      "Message" : "Illegal character in path at index 40: Packages(Id='mysql.odbc',Version='5.1.14\n')"
    }
  }, {
    "name" : "Request",
    "type" : "topic",
    "value" : "Request"
  }, {
    "name" : "Details",
    "type" : "table",
    "value" : {
      "Action" : "GET",
      "path" : "/mysql.odbc/5.1.14\n"
    }
  }, {
    "name" : "Parameters",
    "type" : "table",
    "value" : {
      "describe" : "json"
    }
  }, {
    "name" : "Headers",
    "type" : "table",
    "value" : {
      "Accept" : "*/*",
      "User-Agent" : "python-requests/2.27.1",
      "Connection" : "keep-alive",
      "Host" : "fqdn",
      "Accept-Encoding" : "gzip, deflate"
    }
  }, {
    "name" : "Attributes",
    "type" : "table",
    "value" : {
      "org.apache.shiro.subject.support.DefaultSubjectContext.SESSION_CREATION_ENABLED" : false,
      "Key[type=org.sonatype.nexus.security.SecurityFilter, annotation=[none]].FILTERED" : true,
      "authcAntiCsrf.FILTERED" : true,
      "nx-authc.FILTERED" : true,
      "org.apache.shiro.web.servlet.ShiroHttpServletRequest_SESSION_ID_URL_REWRITING_ENABLED" : true,
      "javax.servlet.include.servlet_path" : "/repository/repo/mysql.odbc/5.1.14
",
      "nx-anonymous.FILTERED" : true,
      "org.sonatype.nexus.security.anonymous.AnonymousFilter.originalSubject" : "org.apache.shiro.web.subject.support.WebDelegatingSubject@33c429ba",
      "nx-apikey-authc.FILTERED" : true
    }
  }, {
    "name" : "Payload",
    "type" : "table",
    "value" : {
      "Content-Type" : "",
      "Size" : -1
    }
  } ]
}

Code with variable of list with in the code:

import requests
import json
import re

repo_name = "repo"

file_list = ["mysql.odbc/5.1.11","mysql.odbc/5.1.14"]
for i in file_list:
   url = "http://fqdn/repository/{0}/{1}?describe=json".format(repo_name, i)
   response = requests.get(url)
   json_data = response.text
   data = json.loads(json_data)
   for size in data['items']:
       if size['name'] == 'Payload':
           value_size= size['value']['Size']
           if value_size != -1:
              print(value_size)

[Expected output]Output with list passed within the code as

{
  "parameters" : {
    "path" : "/mysql.odbc/5.1.14",
    "nexusUrl" : "http://fqdn"
  },
  "items" : [ {
    "name" : "Request",
    "type" : "topic",
    "value" : "Request"
  }, {
    "name" : "Details",
    "type" : "table",
    "value" : {
      "Action" : "GET",
      "path" : "/mysql.odbc/5.1.14"
    }
  }, {
    "name" : "Parameters",
    "type" : "table",
    "value" : {
      "describe" : "json"
    }
  }, {
    "name" : "Headers",
    "type" : "table",
    "value" : {
      "Accept" : "*/*",
      "User-Agent" : "python-requests/2.27.1",
      "Connection" : "keep-alive",
      "Host" : "fqdn",
      "Accept-Encoding" : "gzip, deflate"
    }
  }, {
    "name" : "Attributes",
    "type" : "table",
    "value" : {
      "org.apache.shiro.subject.support.DefaultSubjectContext.SESSION_CREATION_ENABLED" : false,
      "Key[type=org.sonatype.nexus.security.SecurityFilter, annotation=[none]].FILTERED" : true,
      "authcAntiCsrf.FILTERED" : true,
      "nx-authc.FILTERED" : true,
      "org.apache.shiro.web.servlet.ShiroHttpServletRequest_SESSION_ID_URL_REWRITING_ENABLED" : true,
      "javax.servlet.include.servlet_path" : "/repository/repo/mysql.odbc/5.1.14",
      "nx-anonymous.FILTERED" : true,
      "org.sonatype.nexus.security.anonymous.AnonymousFilter.originalSubject" : "org.apache.shiro.web.subject.support.WebDelegatingSubject@1433a6c9",
      "nx-apikey-authc.FILTERED" : true
    }
  }, {
    "name" : "Payload",
    "type" : "table",
    "value" : {
      "Content-Type" : "",
      "Size" : -1
    }
  }, {
    "name" : "Response",
    "type" : "topic",
    "value" : "Response"
  }, {
    "name" : "Status",
    "type" : "table",
    "value" : {
      "Code" : 200,
      "Message" : ""
    }
  }, {
    "name" : "Headers",
    "type" : "table",
    "value" : {
      "ETag" : "\"df4f013db18103f1b9541cdcd6ba8632\"",
      "Content-Disposition" : "attachment; filename=mysql.odbc.5.1.14.nupkg",
      "Last-Modified" : "Tue, 13 Oct 2015 03:54:48 GMT"
    }
  }, {
    "name" : "Attributes",
    "type" : "table",
    "value" : { }
  }, {
    "name" : "Payload",
    "type" : "table",
    "value" : {
      "Content-Type" : "application/zip",
      "Size" : 3369
    }
  } ]
}

I am not sure if I am doing something wrong or missing something simple.

Any help is much appreciated.

CodePudding user response：

It looks like the newlines are being passed into the url string

"Message" : "Illegal character in path at index 40: Packages(Id='mysql.odbc',Version='5.1.14**\n**')"

You can do something like this to remove them

with open('file_list_new.txt') as file:
    for line in file:
       url = "http://fqdn/repository/{0}/{1}?describe=json".format(repo_name, line.strip())