I'm scratching my head. My goal is to get an input of json und put it out into an influxdb. Since' I'm gathering data from a pfSense my tools are limited. Therefor I'm using Python 3.8.
Developing on an Debian Machine first I wrote this code:
#!/usr/local/bin/python3.8
import os
import json
from influxdb import InfluxDBClient
from datetime import datetime
INTERFACES = ["igb0", "pppoe0"]
WORKING_FOLDER = "/tmp/json"
NOW = datetime.now()
CLIENT = InfluxDBClient("host", "8086", "user", "pw", "firewall")
for interface in INTERFACES:
stats = {}
os.popen(
"vnstat --json d 2 -i {} > {}/{}_d.json".format(interface, WORKING_FOLDER, interface)
).read()
os.popen(
"vnstat --json m 2 -i {} > {}/{}_m.json".format(interface, WORKING_FOLDER, interface)
).read()
with open(f"{WORKING_FOLDER}/{interface}_d.json", "r") as j:
day = json.loads(j.read())
with open(f"{WORKING_FOLDER}/{interface}_m.json", "r") as k:
month = json.loads(k.read())
if not len(day["interfaces"][0]["traffic"]["day"]) == 2:
stats["yesterday"] = {"rx": 0, "tx": 0}
stats["today"] = {
"rx": day["interfaces"][0]["traffic"]["day"][0]["rx"],
"tx": day["interfaces"][0]["traffic"]["day"][0]["tx"],
}
else:
stats["yesterday"] = {
"rx": day["interfaces"][0]["traffic"]["day"][0]["rx"],
"tx": day["interfaces"][0]["traffic"]["day"][0]["tx"],
}
stats["today"] = {
"rx": day["interfaces"][0]["traffic"]["day"][1]["rx"],
"tx": day["interfaces"][0]["traffic"]["day"][1]["tx"],
}
if not len(month["interfaces"][0]["traffic"]["month"]) == 2:
stats["last_month"] = {"rx": 0, "tx": 0}
stats["this_month"] = {
"rx": month["interfaces"][0]["traffic"]["month"][0]["rx"],
"tx": month["interfaces"][0]["traffic"]["month"][0]["tx"],
}
else:
stats["last_month"] = {
"rx": month["interfaces"][0]["traffic"]["month"][0]["rx"],
"tx": month["interfaces"][0]["traffic"]["month"][0]["tx"],
}
stats["this_month"] = {
"rx": month["interfaces"][0]["traffic"]["month"][1]["rx"],
"tx": month["interfaces"][0]["traffic"]["month"][1]["tx"],
}
json_body = [
{
"measurement": f"stats_{interface}",
"time": NOW,
"fields": {
"yesterday_rx": stats["yesterday"]["rx"],
"yesterday_tx": stats["yesterday"]["tx"],
"yesterday_total": int(stats["yesterday"]["rx"]) int(stats["yesterday"]["tx"]),
"today_rx": stats["today"]["rx"],
"today_tx": stats["today"]["tx"],
"today_total": int(stats["today"]["rx"]) int(stats["today"]["tx"]),
"last_month_rx": stats["last_month"]["rx"],
"last_month_tx": stats["last_month"]["tx"],
"last_month_total": int(stats["last_month"]["rx"]) int(stats["last_month"]["tx"]),
"this_month_rx": stats["this_month"]["rx"],
"this_month_tx": stats["this_month"]["tx"],
"this_month_total": int(stats["this_month"]["rx"]) int(stats["this_month"]["tx"]),
},
}
]
CLIENT.write_points(json_body)
Testing it on my Debian machine (with line 12 & 13 commented out to not overwrite my json files) everything works like a charm. So I moved the script to my pfSense and I'm getting the following error:
[2.5.2-RELEASE][admin@pfSense]/usr/local/pythonscripts: ./trafficstats.py
Traceback (most recent call last):
File "./trafficstats.py", line 18, in <module>
month = json.loads(k.read())
File "/usr/local/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
And I'm asking myself why. Because looking at the moth file is a valid json if I see correctly:
{"vnstatversion":"2.7","jsonversion":"2","interfaces":[{"name":"igb0","alias":"WAN","created":{"date":{"year":2022,"month":2,"day":4}},"updated":{"date":{"year":2022,"month":2,"day":12},"time":{"hour":9,"minute":30}},"traffic":{"total":{"rx":82416467756,"tx":43825701833},"month":[{"id":88,"date":{"year":2022,"month":2},"rx":82416467756,"tx":43825701833}]}}]}
Does anybody got a hint or idea whats going on?
Cheers, Gamie
CodePudding user response：
"Expecting value at char 0" generally means that you're trying to json.loads() an empty string:
>>> import json
>>> json.loads("")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Python39\lib\json\__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "C:\Python39\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Python39\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
You're not checking whether those vnstat invocations succeed (you should be using subprocess.check_call() instead of os.popen anyway since you're not really even using pipes), so it's likely the files have been created but are empty.
All in all, I'd maybe refactor things to something like this:
- use
subprocess.check_call()to wait for the process to finish (and check result) - don't use a
statsdict that's basically just thrown away - refactor repetitive bits
#!/usr/local/bin/python3.8
import json
import subprocess
from datetime import datetime
from influxdb import InfluxDBClient
INTERFACES = ["igb0", "pppoe0"]
WORKING_FOLDER = "/tmp/json"
NOW = datetime.now()
CLIENT = InfluxDBClient("host", "8086", "user", "pw", "firewall")
# Process an interfaces.X.traffic.PERIOD dict into a pair of (previous_rx, previous_tx), (current_rx, current_tx)
def process_traffic(traffic):
if len(traffic) != 2:
return ((0, 0), (traffic[0]["rx"], traffic[0]["tx"]))
return ((traffic[0]["rx"], traffic[0]["tx"]), (traffic[1]["rx"], traffic[1]["tx"]))
def process_iface(interface):
day_json = f"{WORKING_FOLDER}/{interface}_d.json"
month_json = f"{WORKING_FOLDER}/{interface}_m.json"
# TODO: consider shlex.quote for safety
subprocess.check_call(
f"vnstat --json d 2 -i {interface} > {day_json}",
shell=True,
)
subprocess.check_call(
f"vnstat --json m 2 -i {interface} > {month_json}",
shell=True,
)
with open(day_json, "r") as j:
day_data = json.loads(j.read())
with open(month_json, "r") as k:
month_data = json.loads(k.read())
(yesterday_rx, yesterday_tx), (today_rx, today_tx) = process_traffic(
day_data["interfaces"][0]["traffic"]["day"]
)
(last_month_rx, last_month_tx), (this_month_rx, this_month_tx) = process_traffic(
month_data["interfaces"][0]["traffic"]["month"]
)
json_body = [
{
"measurement": f"stats_{interface}",
"time": NOW,
"fields": {
"yesterday_rx": yesterday_rx,
"yesterday_tx": yesterday_tx,
"yesterday_total": int(yesterday_rx) int(yesterday_tx),
"today_rx": today_rx,
"today_tx": today_tx,
"today_total": int(today_rx) int(today_tx),
"last_month_rx": last_month_rx,
"last_month_tx": last_month_tx,
"last_month_total": int(last_month_rx) int(last_month_tx),
"this_month_rx": this_month_rx,
"this_month_tx": this_month_tx,
"this_month_total": int(this_month_rx) int(this_month_tx),
},
}
]
CLIENT.write_points(json_body)
def main():
for interface in INTERFACES:
process_iface(interface)
if __name__ == "__main__":
main()