Updates README since recent testing, adds changelog

README.md

@@ -53,6 +53,13 @@ Use a simple filename like `stats.json` or a path like `statorio/out.json`. File
 
 Changing the filename setting will delete the old file (if it exists).
 
+**For statsd** formatted metrics use a `.statsd` file extension. The mod will write out lines like this suitable for statsd consumption:
+
+```
+game.players.1.online_time:36709853|g
+game.players.2.online_time:47257460|g
+```
+
 ### Start playing
 
 Play any single player game on your local computer and Statorio will start running. If you save the game then Statorio will become a dependency of your save, and request installation (if its missing) each time the game is loaded. Its safe to remove if you don't want to use it any more.
@@ -161,12 +168,65 @@ Used to get production and consumption total counts, or flow count values for a
 For example `game.forces.player.kill_count_statistics.get_flow_count.*.input.one_hour` for all kills or `game.forces.player.kill_count_statistics.get_flow_count.character.output.one_hour` deaths. `game.pollution_statistics.get_flow_count.*.input.one_hour` for all pollutants on the map.
 
 
+## Realtime graphs
+
+![Ore mining graph](https://factorio.xn--31ab.com/statorio/graph-mining-2.png)
+
+![Production graph](https://factorio.xn--31ab.com/statorio/graph-production.png)
+
+
+There's a lot of ways to make graphs from the Statorio JSON output (NodeRED, Grafana, custom), but I haven't explored them yet to write a guide.
+
+I use [Netdata](https://learn.netdata.cloud/docs/get#install-the-netdata-agent) (minimal, powerful, open source, and cloud account not required) which includes a metrics collector plugin called `statsd`. Netdata is installed on the same server as the Factorio headless server for simplicity but there's no reason it couldn't be elsewhere. Follow the [quick setup guide](https://learn.netdata.cloud/docs/quickstart/single-node) and get it working for the usual server metrics first. Just remember when browsing the docs that you're using the *Netdata Agent* only.. other features need a (free) cloud account.
+
+Configure Statorio to output a filename ending `.statsd` - the file now contains one metric per line in a format that `statsd` can digest (instead of JSON). A simple bash command or similar script can be used to watch the file for changes and stream it to statsd over TCP. Here is a sample command, run it from Factorio's `script-output` directory:
+
+```
+while true; do fswatch -1 stats.statsd | xargs -0 -n1 -I{} sleep 1; ncat --send-only localhost 8125 < stats.statsd; done;
+```
+
+Refresh the Netdata dashboard. You should see simple metric graphs appear under the "statsd" heading. Ugly but we know its working.
+
+Finally the Netdata statsd plugin can be configured with your own [synthetic charts](https://learn.netdata.cloud/docs/agent/collectors/statsd.plugin#synthetic-statsd-charts) to show realtime graphs.
+
+Here's an example:
+
+```
+# /etc/netdata/statsd.d/statorio.conf
+
+[app]
+    name = Factorio
+    metrics = game.*
+    private charts = yes
+    gaps when not collected = no
+    memory mode = ram
+
+[player.production.raw_material]
+    title = Plates
+    family = Production
+    context = game.production.raw_material
+    units = items/m
+    type = area
+    dimension = game.forces.player.item_production_statistics.get_flow_count.iron-plate.input.one_minute 'Iron'
+    dimension = game.forces.player.item_production_statistics.get_flow_count.copper-plate.input.one_minute 'Copper'
+    dimension = game.forces.player.item_production_statistics.get_flow_count.steel-plate.input.one_minute 'Steel'
+
+```
+
+Restart Netdata with the new configuration `sudo systemctl restart netdata` and now you should have realtime graphs.
+
+Graphs are updated as often as statsd receives metrics, which in theory should be the number of game ticks you set in Statorio mod settings. I have it at 60-180 ticks (approx 1-3 seconds) on a moderate size base with no problems.
+
+Now you can create more synthetic graphs for the metrics you want to watch, group them in a logical way, and even create Netdata alarms for your factory.
+
+It's even possible to make [custom dashboards](https://learn.netdata.cloud/docs/agent/web/gui/custom) using just basic HTML and the included `dashboard.js` library. Create gauges, charts, sparklines, pie charts and more. Super 📈
+
+
 ## Exploring the vanilla game API
 
 If you can't find what you need in Factorio's game API then here are some clues to what's available.
 
 
-
 ### Game
 
 Where almost everything happens. Accessed with `game.`. See [API docs for LuaGameScript](https://lua-api.factorio.com/latest/LuaGameScript.html).
@@ -260,7 +320,7 @@ Pull requests for any of these are very welcome:
 
 - [x] Console commands for admins only
 - [x] Cached settings for performance
-- [ ] Check for multiplayer desyncs
+- [x] Check for multiplayer desyncs
 - [ ] More game API functions exposed
 - [ ] Robust API calling methods
 - [ ] Multiplayer settings
@@ -271,8 +331,10 @@ Pull requests for any of these are very welcome:
   - [ ] LTN
   - [ ] ...
 - [ ] Extremely basic UI output
-- [ ] More output file formats beyond JSON
+- [X] More output file formats beyond JSON
+- [ ] Support for simulations
 - [ ] Translations of the mod and supporting documentation
+- [ ] Sanitise game speed vs tick output
 
 If you'd like to add anything else then its worth reaching out first.
 

changelog.txt

@@ -0,0 +1,21 @@
+---------------------------------------------------------------------------------------------------
+Version: 1.0.3
+Date: 15.05.2022
+  Fixes:
+    - Fixes bug where get_entity_count always returned an API error
+    - get_flow_count now supports LuaItemProductionFlowStatistics and other undocumented parents
+  Features:
+    - Improved help prompts for missing command arguments
+    - Added `rm` as an alias of `del`
+    - Added `show` as an alias of `list`
+---------------------------------------------------------------------------------------------------
+Version: 1.0.2
+Date: 11.02.2021
+  Features:
+    - Added statsd support
+    - Improved examples in documentation
+---------------------------------------------------------------------------------------------------
+Version: 1.0.1
+Date: 09.02.2021
+  Features:
+    - First release

info.json

@@ -1,11 +1,11 @@
 {
   "name": "statorio",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "title": "Statorio",
-  "author": "aoi44",
+  "author": "chksm",
   "homepage": "https://factorio.яю.com/statorio/",
   "contact": "a@яю.com",
   "factorio_version": "1.1",
   "dependencies": ["base >= 1.0"],
-  "description": "Experimental! Export realtime JSON statistics to disk. Use the stat command to add, list and remove game API metrics. See homepage for usage"
-}
+  "description": "Export realtime JSON statistics to disk from the built-in Factorio game API. Use the /stat command to add, list and remove game API metrics. See homepage for usage"
+}

script/commands.lua

@@ -21,7 +21,7 @@ commands.add_command("stat", "Used to control metrics in Statorio mod\n[add|list
 
   argsArr = split(param.parameter, " ")
 
-  if argsArr[1] == "list" then
+  if argsArr[1] == "list" or argsArr[1] == "show" then
 
     if table_size(global.statorio.metrics) > 0 then
       for index, val in pairs(global.statorio.metrics) do
@@ -33,16 +33,31 @@ commands.add_command("stat", "Used to control metrics in Statorio mod\n[add|list
 
   elseif argsArr[1] == "add" then
 
+    if not argsArr[2] then
+      player.print("Adding requires a metric")
+      return
+    end
+
     table.insert(global.statorio.metrics, argsArr[2])
     player.print("Added metric ["..table_size(global.statorio.metrics).."]")
 
-  elseif argsArr[1] == "del" then
+  elseif argsArr[1] == "del" or argsArr[1] == "rm" then
+
+    if not argsArr[2] then
+      player.print("Deleting requires the index number of a metric from the list command")
+      return
+    end
 
     table.remove(global.statorio.metrics, argsArr[2])
     player.print("Removed metric ["..argsArr[2].."]")
 
   elseif argsArr[1] == "test" then
 
+    if not argsArr[2] then
+      player.print("Testing requires a metric")
+      return
+    end
+
     if not Fetcher.traverseApiForValues(argsArr[2], function(name, val)
         player.print(name..": "..safeString(val))
       end)
@@ -53,5 +68,6 @@ commands.add_command("stat", "Used to control metrics in Statorio mod\n[add|list
   else
 
     player.print("Try list, add, del or test. For example:\n/stat list")
+    player.print("For more information see https://mods.factorio.com/mod/statorio")
   end
 end)

script/fetcher.lua

@@ -17,7 +17,9 @@ this.overlays.get_entity_count = function(parent, entity)
     return parent.get_entity_count(entity)
   end)
 
-  return data
+  if res then return data end
+
+  return nil
 
 end
 
@@ -45,10 +47,17 @@ this.callOverlayFunction = function(functionName, callback, name, parent, args)
 
     --data = parent.get_entity_count(args[1])
     data = this.overlays.get_entity_count(parent, args[1])
-    this.doCallback(callback, name, data)
+
+    if data == nil then
+      return false
+    else
+      this.doCallback(callback, name, data)
+      return true
+    end
 
   -- flow statistics
-  elseif functionName == "get_flow_count" and parent.object_name == "LuaFlowStatistics" then
+  -- worst hack to validate that parent inherits from FlowStatistics
+  elseif functionName == "get_flow_count" and string.sub(parent.object_name, -14) == "FlowStatistics" then
 
     if args[2] == 'input' then isInput = true else isInput = false end
 
@@ -75,6 +84,11 @@ this.callOverlayFunction = function(functionName, callback, name, parent, args)
 
     end
 
+  elseif functionName == "get_flow_count" then
+
+    game.players[1].print("Cannot get_flow_count for unexpected instance of:")
+    game.players[1].print(parent.object_name)
+
   end
 
 end

script/publisher.lua

@@ -33,21 +33,38 @@ end
 
 -- publish stored metrics
 this.publish = function()
+  if get_file_extension(this.filename) == "statsd" then
+    this.publishStatsd()
+  else
+    this.publishJson()
+  end
+
+  -- reset local cache
+  this.metrics = {}
+end
+
+this.publishStatsd = function()
+  -- generate lines of "metric_name:value|g"
+  statsdString = ""
+  for key, val in pairs(this.metrics) do
+    if type(val) == "string" then
+      statsdString = statsdString..key..":"..val.."|s\n"
+    else
+      statsdString = statsdString..key..":"..val.."|g\n"
+    end
+  end
 
+  game.write_file(this.filename, statsdString)
+end
+
+this.publishJson = function()
   -- generate json
   jsonString = game.table_to_json(this.metrics)
 
   -- write to disk
   game.write_file(this.filename, jsonString)
-
-  -- clear json
-  jsonString = nil
-
-  -- reset local cache
-  this.metrics = {}
 end
 
-
 this.setFilename = function(filename)
 
   if this.filename ~= nil then

script/utilities.lua

@@ -45,8 +45,13 @@ function checkTableKeyExists(tableName, needle)
 end
 
 function isNumeric(x)
-    if tonumber(x) ~= nil then
-        return true
-    end
-    return false
+  if tonumber(x) ~= nil then
+      return true
+  end
+  return false
+end
+
+function get_file_extension(path)
+  parts = split(path, ".")
+  return parts[#parts]
 end