some timezone corrections to UTC, write readme

This commit is contained in:
2023-07-06 09:44:37 -07:00
parent ee0c28d2c8
commit c250310d58
10 changed files with 281 additions and 107 deletions

View File

@@ -55,6 +55,15 @@ addMissionEventHandler ["ExtensionCallback", {
]
];
missionNamespace setVariable ["AttendanceTracker_DBConnected", true];
};
};
case "writeWorldInfo": {
if (_response#0 == "WORLD_ID") then {
AttendanceTracker_worldId = _response#1;
// world info written. mission info depends on that, so now we'll write it
// log mission info and get back the row Id to send with future messages
private _response = "AttendanceTracker" callExtension [
"logMission",
@@ -62,12 +71,11 @@ addMissionEventHandler ["ExtensionCallback", {
[AttendanceTracker getVariable ["missionContext", createHashMap]] call CBA_fnc_encodeJSON
]
];
missionNamespace setVariable ["AttendanceTracker_DBConnected", true];
};
};
case "writeMission": {
if (_response#0 == "MISSION_ID") then {
// mission has written so lets finish out init and set missionId for the returned PK, activating the ability for attendance records to send.
AttendanceTracker_missionId = _response#1;
};
};

View File

@@ -40,6 +40,19 @@ AttendanceTracker setVariable ["missionContext", createHashMapFromArray [
AttendanceTracker setVariable ["allUsers", createHashMap];
AttendanceTracker setVariable ["rowIds", createHashMap];
// update the extension with the current server time to identify restarts
[
{
'AttendanceTracker' callExtension [
"updateServerTime",
[
round(diag_tickTime)
]
]
},
30
] call CBA_fnc_addPerFrameHandler;
{
if (!isServer) exitWith {};
_x params ["_ehName", "_code"];

View File

@@ -13,7 +13,7 @@ systemTimeUTC apply {if (_x < 10) then {"0" + str _x} else {str _x}} params [
];
format[
"%1-%2-%3 %4:%5:%6",
"%1-%2-%3T%4:%5:%6.000Z",
_year,
_month,
_day,

View File

@@ -19,7 +19,10 @@ _hash set ["isJIP", _isJIP];
_hash set ["roleDescription", _roleDescription];
[
{missionNamespace getVariable ["AttendanceTracker_DBConnected", false]},
{
missionNamespace getVariable ["AttendanceTracker_DBConnected", false] &&
missionNamespace getVariable ["AttendanceTracker_missionId", -1] > 0
},
{"AttendanceTracker" callExtension ["writeAttendance", [[_this] call CBA_fnc_encodeJSON]]},
_hash, // args
30 // timeout in seconds. if DB never connects, we don't want these building up

View File

@@ -19,7 +19,10 @@ _hash set ["isJIP", _isJIP];
_hash set ["roleDescription", _roleDescription];
[
{missionNamespace getVariable ["AttendanceTracker_DBConnected", false]},
{
missionNamespace getVariable ["AttendanceTracker_DBConnected", false] &&
missionNamespace getVariable ["AttendanceTracker_missionId", -1] > 0
},
{"AttendanceTracker" callExtension ["writeDisconnectEvent", [[_this] call CBA_fnc_encodeJSON]]},
_hash, // args
30 // timeout in seconds. if DB never connects, we don't want these building up

View File

@@ -8,6 +8,8 @@
},
"armaConfig": {
"dbUpdateIntervalSeconds": 90,
"serverEventFillNullMinutes": 90,
"missionEventFillNullMinutes": 15,
"debug": false
}
}

198
README.md
View File

@@ -1,83 +1,157 @@
# Arma 3 Attendance Tracker
---
## Setup
### Set Up a Database Engine
**You will need a running MySQL or MariaDB instance.**
The following SQL commands will set up the necessary tables for the application. You can run them from the MySQL command line or from a tool like phpMyAdmin.
If you do not have a MySQL or MariaDB instance, you can use one of the following:
*In future, an ORM will be used to set this up automatically.*
- [MySQL Community Server](https://dev.mysql.com/downloads/mysql/) (free)
- [MariaDB](https://mariadb.org/) (free)
- [Docker](https://www.docker.com/) (free)
- [MySQL Docker Image](https://hub.docker.com/_/mysql)
- [MariaDB Docker Image](https://hub.docker.com/_/mariadb)
The tested and recommended database engine is MariaDB version 11.0.2.
### Create a Database
You will need to create a database in your instance. You can do this using your preferred MySQL client or one of the following:
- [phpMyAdmin](https://www.phpmyadmin.net/) (web based)
- [HeidiSQL](https://www.heidisql.com/) (Windows only and simpler)
- [MySQL Workbench](https://www.mysql.com/products/workbench/) (feature rich but heavy)
- [DBeaver](https://dbeaver.io/) (feature rich but heavy)
You could also use a CLI client such as the `mysql` command line client to run the following command:
```sql
CREATE DATABASE `arma3_attendance` /*!40100 DEFAULT CHARACTER SET utf8mb3 */;
USE `arma3_attendance`;
-- a3server.missions definition
CREATE TABLE `missions` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`world_id` int(11) DEFAULT NULL,
`mission_hash` varchar(100) NOT NULL DEFAULT '',
`mission_name` varchar(100) NOT NULL,
`mission_name_source` varchar(100) DEFAULT NULL,
`briefing_name` varchar(100) DEFAULT NULL,
`on_load_name` varchar(100) DEFAULT NULL,
`author` varchar(100) DEFAULT NULL,
`server_name` varchar(100) DEFAULT NULL,
`server_profile` varchar(100) DEFAULT NULL,
`mission_start` datetime DEFAULT NULL COMMENT 'In UTC',
PRIMARY KEY (`id`),
KEY `mission_hash` (`mission_hash`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb3;
-- arma3_attendance.attendance definition
CREATE TABLE `attendance` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`join_time` datetime DEFAULT NULL COMMENT 'Stored in UTC',
`disconnect_time` datetime DEFAULT NULL COMMENT 'Stored in UTC',
`mission_hash` varchar(100) DEFAULT NULL,
`event_type` varchar(100) NOT NULL,
`player_id` varchar(30) NOT NULL,
`player_uid` varchar(100) NOT NULL,
`profile_name` varchar(100) NOT NULL,
`steam_name` varchar(100) DEFAULT NULL,
`is_jip` tinyint(4) DEFAULT NULL,
`role_description` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
KEY `mission_hash` (`mission_hash`),
CONSTRAINT `attendance_ibfk_1` FOREIGN KEY (`mission_hash`) REFERENCES `missions` (`mission_hash`) ON UPDATE CASCADE
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb3;
-- a3server.worlds definition
CREATE TABLE `worlds` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`author` varchar(100) DEFAULT NULL,
`display_name` varchar(100) DEFAULT NULL,
`world_name` varchar(100) NOT NULL,
`world_name_original` varchar(100) DEFAULT NULL,
`world_size` int(11) DEFAULT NULL,
`latitude` float DEFAULT NULL,
`longitude` float DEFAULT NULL,
`workshop_id` varchar(50) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `world_name` (`world_name`)
) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8mb3;
CREATE DATABASE `arma3_attendance`;
```
Finally, copy `config.example.json` to `config.json` and update it with your database credentials and path.
### Mod Installation
## QUERIES
1. Download the latest release from the [releases page](https://github.com/indig0fox/Arma3-AttendanceTracker/releases).
1. Extract the .zip and move `@AttendanceTracker` to your Arma 3 server's root directory.
1. Inside of `@AttendanceTracker` you will find a `config.json` file. Open this file and configure it to your circumstances. See the [Configuration](#configuration) section for more information.
1. Add the mod to your server's startup parameters. For example: `-serverMod="@AttendanceTracker;"`
At next run, the Arma 3 server will launch with the mod running.
### Configuration
The configuration file example is located at `@AttendanceTracker/config.json.example`. This should be copied to `@AttendanceTracker/config.json` and edited to suit your circumstances.
The following table describes the configuration options.
| Key | Type | Description | Default |
| --- | --- | --- | --- |
| sqlConfig.mySqlHost | string | The hostname of your MySQL instance. | localhost |
| sqlConfig.mySqlPort | integer | The port of your MySQL instance. | 3306 |
| sqlConfig.mySqlUser | string | The username to use when connecting to your MySQL instance. | root |
| sqlConfig.mySqlPassword | string | The password to use when connecting to your MySQL instance. | root |
| sqlConfig.mySqlDatabase | string | The name of the database to use. | arma3_attendance |
| armaConfig.dbUpdateIntervalSeconds | integer | The number of seconds between disconnect_time updates per user. | 90 |
| armaConfig.serverEventFillNullMinutes | integer | The max session duration to fill in for missing server disconnect_time values. | 90 |
| armaConfig.missionEventFillNullMinutes | integer | The max session duration to fill in for missing mission disconnect_time values. | 15 |
| armaConfig.debug | boolean | Whether or not to enable debug logging. | false |
## Usage
The extension uses [GORM](https://gorm.io/) for database access and will automatically create the schema in the database you specify in the configuration file.
---
## Important Notes
### Logging
"debug": true:
The extension will log ERROR and WARN events to the Arma 3 server's RPT file, which can be found in the server's profile folder.
"debug": false:
The extension will log ERROR, WARN, INFO, and DEBUG events to the Arma 3 server's RPT file, which can be found in the server's profile folder.
All events will be always be logged to `@AttendanceTracker/attendanceTracker.log` in log line format.
### Timezone
All times will be logged as UTC time. This is to ensure that all times are logged in a consistent manner, regardless of the timezone of the server. Because these are DATETIME fields, they will not be adjusted to your local time when viewing them in a database client.
To do so, you can use the following function in your queries:
```sql
CONVERT_TZ(<field>, 'UTC', 'US/Eastern')
```
A full list of timezones available to your database can be found this way:
```sql
SELECT *
FROM mysql.time_zone_name
```
### Performance
The extension will update the disconnect_time field for each player every `dbUpdateIntervalSeconds` seconds. This is to ensure that the disconnect_time field is updated in the event that the server crashes or the mission ends without a disconnect event.
These calls are threaded in the Go runtime and will not block the Arma 3 server while processing. The default value of 90 seconds should be sufficient for most servers. Each period begins when a player connects to the server or connects to a mission, which provides a natural offset.
### NULL disconnect_time Values
In the event that the server crashes or a disconnect event for a mission is not sent, the next join for each will update past rows based on the following:
If the join time for a row is within [`{event_type}EventFillNullMinutes`](#configuration) minutes of the previous disconnect time, the previous disconnect time will be updated to the new join time. Otherwise, it will be set as [`{event_type}EventFillNullMinutes`](#configuration) from the join time for that row.
This is an attempt to account for missing events for individual players while not attributing large gap periods to their calculated session times. If the server crashes, the extension will update all rows with a NULL disconnect_time to the current time. See [Server Crash Time Filling](#server-crash-time-filling) for more information.
#### Server Crash Time Filling
The addon will update `@AttendanceTracker/lastServerTime.txt` with Arma 3's `diag_tickTime` every 30 seconds. This is to ensure that the server time is always available to the extension, even if the server crashes. This file is not used for any other purpose.
On each time update, the extension will check this file and compare the received value to it. If the lastServerTime < lastServerTime.txt, the extension will assume that the server has restarted and will update all event rows with a NULL disconnect_time to the current time OR the threshold specified in the configuration file, whichever produces the smaller session duration.
---
## Database Schema
| Table Name | Description |
| --- | --- |
| worlds | Stores world information. |
| missions | Stores mission information. |
| attendance_items | Stores rows that indicate player information and join/disconnect times. |
### Worlds
The worlds table will store basic info about the world. This is used to link missions to worlds.
### Missions
The missions table will store basic info about the mission. This is used to link attendance items to missions.
### Attendance Items
The attendance_items table will store rows that indicate player information and join/disconnect times. This can be used to calculate play time per player per mission. Each row is also linked to a mission, so that these records can be grouped.
---
## Useful Queries
### Show missions with attendance
This will retrieve a view showing all missions with attendance data, sorted by the most recent mission joins first. Mission events without a mission disconnect_time (due to server crash or in-progress mission) will be ignored.
See [Timezone](#timezone) for more information on converting times to your local timezone.
```sql
select
a.server_profile as Server,
a.briefing_name as "Mission Name",
a.mission_start as "Start Time",
CONVERT_TZ(a.mission_start, 'UTC', 'US/Eastern') as "Start Time",
b.display_name as "World",
c.profile_name as "Player Name",
c.player_uid as "Player UID",
@@ -86,11 +160,11 @@ select
c.join_time,
c.disconnect_time
) as "Play Time (m)",
c.join_time as "Join Time",
c.disconnect_time as "Leave Time"
CONVERT_TZ(c.join_time, 'UTC', 'US/Eastern') as "Join Time",
CONVERT_TZ(c.disconnect_time, 'UTC', 'US/Eastern') as "Leave Time"
from missions a
LEFT JOIN worlds b ON a.world_id = b.id
LEFT JOIN attendance c ON a.mission_hash = c.mission_hash
LEFT JOIN attendance_items c ON a.mission_hash = c.mission_hash
where
c.event_type = 'Mission'
AND c.disconnect_time IS NOT NULL

View File

@@ -12,11 +12,13 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"path"
"reflect"
"runtime"
"strconv"
"strings"
"time"
"unsafe"
@@ -33,6 +35,7 @@ var extensionCallbackFnc C.extensionCallback
var ADDON_FOLDER string = getDir() + "\\@AttendanceTracker"
var LOG_FILE string = ADDON_FOLDER + "\\attendanceTracker.log"
var CONFIG_FILE string = ADDON_FOLDER + "\\config.json"
var SERVER_TIME_FILE string = ADDON_FOLDER + "\\lastServerTime.txt"
var ATTENDANCE_TABLE string = "attendance"
var MISSIONS_TABLE string = "missions"
@@ -49,6 +52,8 @@ var A3Config ArmaConfig
type ArmaConfig struct {
DBUpdateIntervalSeconds int `json:"dbUpdateIntervalSeconds"`
Debug bool `json:"debug"`
ServerEventFillNullMinutes int `json:"serverEventFillNullMinutes"`
MissionEventFillNullMinutes int `json:"missionEventFillNullMinutes"`
}
type ATSQLConfig struct {
@@ -159,7 +164,7 @@ func getMissionHash() string {
functionName := "getMissionHash"
// get md5 hash of string
// https://stackoverflow.com/questions/2377881/how-to-get-a-md5-hash-from-a-string-in-golang
hash := md5.Sum([]byte(time.Now().UTC().Format("2006-01-02 15:04:05")))
hash := md5.Sum([]byte(time.Now().Format("2006-01-02 15:04:05")))
// convert to string
hashString := fmt.Sprintf(`%x`, hash)
@@ -169,26 +174,82 @@ func getMissionHash() string {
func updateServerTime(serverTime uint64) {
functionName := "updateServerTime"
var err error
// check .txt file for server time
// first, check if it exists
if _, err := os.Stat(SERVER_TIME_FILE); os.IsNotExist(err) {
// file does not exist, create it and write serverTime to it
writeLog(functionName, `["Server time file does not exist, creating it", "DEBUG"]`)
err = ioutil.WriteFile(SERVER_TIME_FILE, []byte(strconv.FormatUint(serverTime, 10)), 0666)
if err != nil {
writeLog(functionName, fmt.Sprintf(`["Error writing server time to file: %v", "ERROR"]`, err))
}
return
}
// file exists, read it
line, err := ioutil.ReadFile(SERVER_TIME_FILE)
if err != nil {
writeLog(functionName, fmt.Sprintf(`["Error reading server time file: %v", "ERROR"]`, err))
return
}
// convert to uint64
LAST_SERVER_TIME, err := strconv.ParseUint(string(line), 10, 64)
if err != nil {
writeLog(functionName, fmt.Sprintf(`["Error converting server time to uint64: %v", "ERROR"]`, err))
return
}
// if serverTime is less than last server time, close server events
if serverTime < LAST_SERVER_TIME {
writeLog(functionName, `["Server has restarted, closing pending server sessions in attendance", "INFO"]`)
closeServerEvents()
}
LAST_SERVER_TIME = serverTime
// write server time to file
err = ioutil.WriteFile(SERVER_TIME_FILE, []byte(strconv.FormatUint(serverTime, 10)), 0666)
if err != nil {
writeLog(functionName, fmt.Sprintf(`["Error writing server time to file: %v", "ERROR"]`, err))
return
}
}
func closeServerEvents() {
functionName := "closeServerEvents"
writeLog(functionName, `["Closing server events", "INFO"]`)
// get all server events with null DisconnectTime & set DisconnectTime to current time
op := db.Model(&AttendanceItem{}).Where(`event_type = ? AND disconnect_time IS NULL`).Update("disconnect_time", time.Now().UTC())
if op.Error != nil {
writeLog(functionName, fmt.Sprintf(`["%s", "ERROR"]`, op.Error))
writeLog(functionName, `["Filling missing disconnect events due to server restart.", "DEBUG"]`)
// get all events with null DisconnectTime & set DisconnectTime to current time
var events []AttendanceItem
db.Where("disconnect_time = '0000-00-00 00:00:00'").Find(&events)
for _, event := range events {
// if difference between JoinTime and current time is greater than threshold, set to threshold
if event.EventType == "Server" {
var timeThreshold time.Time = event.JoinTime.Add(-time.Duration(A3Config.ServerEventFillNullMinutes) * time.Minute)
if event.JoinTime.Before(timeThreshold) {
event.DisconnectTime = timeThreshold
} else {
event.DisconnectTime = time.Now()
}
} else if event.EventType == "Mission" {
var timeThreshold time.Time = event.JoinTime.Add(-time.Duration(A3Config.MissionEventFillNullMinutes) * time.Minute)
if event.JoinTime.Before(timeThreshold) {
event.DisconnectTime = timeThreshold
} else {
event.DisconnectTime = time.Now()
}
}
db.Save(&event)
if db.Error != nil {
writeLog(functionName, fmt.Sprintf(`["Error filling missing disconnects: %v", "ERROR"]`, db.Error))
return
}
}
// log how many
writeLog(functionName, fmt.Sprintf(`["%d server events closed", "INFO"]`, op.RowsAffected))
writeLog(functionName, fmt.Sprintf(`["%d missing disconnects filled.", "INFO"]`, len(events)))
}
func connectDB() error {
@@ -199,7 +260,7 @@ func connectDB() error {
// connect to database
var err error
dsn := fmt.Sprintf(
"%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local",
"%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True",
ATConfig.MySQLUser,
ATConfig.MySQLPassword,
ATConfig.MySQLHost,
@@ -271,6 +332,7 @@ func writeWorldInfo(worldInfo string) {
// write world if not exist
var world World
var returnId uint
db.Where("world_name = ?", wi.WorldName).First(&world)
if world.ID == 0 {
writeLog(functionName, `["World not found, writing new world", "INFO"]`)
@@ -280,10 +342,14 @@ func writeWorldInfo(worldInfo string) {
return
}
writeLog(functionName, fmt.Sprintf(`["World written with ID %d", "INFO"]`, wi.ID))
returnId = wi.ID
} else {
// return ID
writeLog(functionName, fmt.Sprintf(`["World exists with ID %d", "INFO"]`, world.ID))
returnId = world.ID
}
writeLog(functionName, fmt.Sprintf(`["WORLD_ID", %d]`, returnId))
}
type Mission struct {
@@ -295,7 +361,7 @@ type Mission struct {
Author string `json:"author"`
ServerName string `json:"serverName"`
ServerProfile string `json:"serverProfile"`
MissionStart string `json:"missionStart"`
MissionStart time.Time `json:"missionStart" gorm:"type:datetime"`
MissionHash string `json:"missionHash"`
WorldName string `json:"worldName" gorm:"-"`
WorldID uint
@@ -384,22 +450,16 @@ func writeDisconnectEvent(data string) {
var attendanceRows []AttendanceItem
db.Where("player_uid = ? AND event_type = ? AND disconnect_time = '0000-00-00 00:00:00'", event.PlayerUID, event.EventType).Find(&attendanceRows)
for _, row := range attendanceRows {
// put to json
json, err := json.Marshal(row)
if err != nil {
writeLog(functionName, fmt.Sprintf(`["%s", "ERROR"]`, err))
return
}
writeLog(functionName, fmt.Sprintf(`["Updating disconnect time for %s", "INFO"]`, json))
if row.JoinTime.Before(time.Now().UTC().Add(-1*time.Hour)) && row.EventType == "Mission" {
// update disconnect time
if row.JoinTime.Before(time.Now().Add(-1*time.Hour)) && row.EventType == "Mission" {
// if mission JoinTime is more than 1 hour ago, simplify this to write DisconnectTime as 1 hour from JoinTime. this to account for crashes where people don't immediately rejoin
row.DisconnectTime = time.Now().UTC().Add(-1 * time.Hour)
} else if row.JoinTime.Before(time.Now().UTC().Add(-6*time.Hour)) && row.EventType == "Server" {
row.DisconnectTime = row.JoinTime.Add(-1 * time.Hour)
} else if row.JoinTime.Before(time.Now().Add(-6*time.Hour)) && row.EventType == "Server" {
// if server JoinTime is more than 6 hours ago, simplify this to write DisconnectTime as 6 hours from JoinTime. this to account for server crashes where people don't immediately rejoin without overwriting valid (potentially lengthy) server sessions
row.DisconnectTime = time.Now().UTC().Add(-6 * time.Hour)
row.DisconnectTime = row.JoinTime.Add(-6 * time.Hour)
} else {
// otherwise, update DisconnectTime to now
row.DisconnectTime = time.Now().UTC()
row.DisconnectTime = time.Now()
}
db.Save(&row)
}
@@ -445,7 +505,7 @@ func writeAttendance(data string) {
} else {
// insert new row
event.JoinTime = time.Now().UTC()
event.JoinTime = time.Now()
row := db.Create(&event)
if row.Error != nil {
writeLog(functionName, fmt.Sprintf(`["%s", "ERROR"]`, row.Error))
@@ -469,14 +529,14 @@ func writeAttendance(data string) {
db.Where("player_id = ? AND player_uid = ? AND event_type = ? AND mission_hash = ?", event.PlayerId, event.PlayerUID, event.EventType, event.MissionHash).Order("join_time desc").First(&attendance)
if attendance.ID != 0 {
// update disconnect time
row := db.Model(&attendance).Update("disconnect_time", time.Now().UTC())
row := db.Model(&attendance).Update("disconnect_time", time.Now())
if row.Error != nil {
writeLog(functionName, fmt.Sprintf(`["%s", "ERROR"]`, row.Error))
return
}
rowId, playerUid = attendance.ID, attendance.PlayerUID
} else {
event.JoinTime = time.Now().UTC()
event.JoinTime = time.Now()
// insert new row
row := db.Create(&event)
if row.Error != nil {
@@ -532,11 +592,22 @@ func goRVExtensionArgs(output *C.char, outputsize C.size_t, input *C.char, argv
}
case "logMission":
if argc == 1 {
writeMission(out[0])
go writeMission(out[0])
}
case "logWorld":
if argc == 1 {
writeWorldInfo(out[0])
go writeWorldInfo(out[0])
}
case "updateServerTime":
if argc == 1 {
// convert to uint64
serverTime, err := strconv.ParseUint(out[0], 10, 64)
if err != nil {
writeLog("updateServerTime", fmt.Sprintf(`["%s", "ERROR"]`, err))
temp = "ERROR parsing server time"
} else {
go updateServerTime(serverTime)
}
}
}
@@ -568,7 +639,7 @@ func callBackExample() {
func getTimestamp() string {
// get the current unix timestamp in nanoseconds
// return time.Now().Local().Unix()
return time.Now().UTC().Format("2006-01-02 15:04:05")
return time.Now().Format("2006-01-02 15:04:05")
}
func trimQuotes(s string) string {