fix: hash RAW cell text for stable IDs, add rawIssue to hash, use DateTime.MinValue not DateTime.Now for unparseable dates

This commit is contained in:
2026-05-09 14:17:36 +10:00
parent 0879dae292
commit d776001b35
+40 -76
View File
@@ -28,7 +28,6 @@ namespace Disco.Plugins.ServiceTracker.Services
} }
string csvData = null; string csvData = null;
if (File.Exists(_cachePath)) if (File.Exists(_cachePath))
{ {
var cacheAge = DateTime.Now - File.GetLastWriteTime(_cachePath); var cacheAge = DateTime.Now - File.GetLastWriteTime(_cachePath);
@@ -46,50 +45,25 @@ namespace Disco.Plugins.ServiceTracker.Services
url = "https://docs.google.com/spreadsheets/d/e/" + id + "/pub?output=csv&gid=" + _config.GId; url = "https://docs.google.com/spreadsheets/d/e/" + id + "/pub?output=csv&gid=" + _config.GId;
else else
url = "https://docs.google.com/spreadsheets/d/" + id + "/export?format=csv&gid=" + _config.GId; url = "https://docs.google.com/spreadsheets/d/" + id + "/export?format=csv&gid=" + _config.GId;
using (var client = new WebClient()) using (var client = new WebClient())
{ {
client.Encoding = Encoding.UTF8; client.Encoding = Encoding.UTF8;
client.Headers.Add("User-Agent", "Mozilla/5.0 DiscoServiceTracker/1.0"); client.Headers.Add("User-Agent", "Mozilla/5.0 DiscoServiceTracker/1.0");
csvData = client.DownloadString(url); csvData = client.DownloadString(url);
} }
if (csvData != null && csvData.TrimStart().StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase)) if (csvData != null && csvData.TrimStart().StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase))
{ {
if (File.Exists(_cachePath)) if (File.Exists(_cachePath)) { csvData = File.ReadAllText(_cachePath); result.Warning = "Got HTML not CSV. Using cache."; }
{ else { result.Error = "Google returned HTML not CSV. Publish as CSV."; return result; }
csvData = File.ReadAllText(_cachePath);
result.Warning = "Got HTML instead of CSV. Using cache.";
}
else
{
result.Error = "Google returned HTML not CSV. Publish as CSV.";
return result;
}
}
else
{
File.WriteAllText(_cachePath, csvData);
} }
else File.WriteAllText(_cachePath, csvData);
} }
catch (WebException ex) catch (WebException ex)
{ {
if (File.Exists(_cachePath)) if (File.Exists(_cachePath)) { csvData = File.ReadAllText(_cachePath); result.Warning = "Using cache. " + ex.Message; }
{ else { result.Error = "Fetch failed: " + ex.Message; return result; }
csvData = File.ReadAllText(_cachePath);
result.Warning = "Using cache. Error: " + ex.Message;
}
else
{
result.Error = "Fetch failed: " + ex.Message;
return result;
}
}
catch (Exception ex)
{
result.Error = "Error: " + ex.Message;
return result;
} }
catch (Exception ex) { result.Error = "Error: " + ex.Message; return result; }
} }
try try
@@ -98,11 +72,7 @@ namespace Disco.Plugins.ServiceTracker.Services
if (result.Tickets.Count == 0 && string.IsNullOrEmpty(result.Warning)) if (result.Tickets.Count == 0 && string.IsNullOrEmpty(result.Warning))
result.Warning = "Sheet OK but no open tickets found."; result.Warning = "Sheet OK but no open tickets found.";
} }
catch (Exception ex) catch (Exception ex) { result.Error = "Parse error: " + ex.Message; }
{
result.Error = "Parse error: " + ex.Message;
}
return result; return result;
} }
@@ -116,28 +86,26 @@ namespace Disco.Plugins.ServiceTracker.Services
{ {
var fields = ParseCsvLine(lines[i]); var fields = ParseCsvLine(lines[i]);
if (fields.Count == 0) continue; if (fields.Count == 0) continue;
bool allEmpty = true; bool allEmpty = true;
for (int j = 0; j < fields.Count; j++) for (int j = 0; j < fields.Count; j++) if (!string.IsNullOrWhiteSpace(fields[j])) { allEmpty = false; break; }
if (!string.IsNullOrWhiteSpace(fields[j])) { allEmpty = false; break; }
if (allEmpty) continue; if (allEmpty) continue;
var taskTitle = SafeGet(fields, _config.ColTask); // Get RAW cell values for stable hashing (before any parsing)
var issueDesc = SafeGet(fields, _config.ColIssue); var rawTimestamp = RawGet(fields, _config.ColTimestamp);
var requestedBy = SafeGet(fields, _config.ColRequestedBy); var rawEmail = RawGet(fields, _config.ColEmail);
var email = SafeGet(fields, _config.ColEmail); var rawTask = RawGet(fields, _config.ColTask);
var timestamp = SafeGetDate(fields, _config.ColTimestamp); var rawIssue = RawGet(fields, _config.ColIssue);
var ticket = new ExternalTicket var ticket = new ExternalTicket
{ {
Source = "ntt", Source = "ntt",
Timestamp = timestamp, Timestamp = SafeGetDate(fields, _config.ColTimestamp),
RequesterEmail = email, RequesterEmail = SafeGet(fields, _config.ColEmail),
RequesterName = requestedBy, RequesterName = SafeGet(fields, _config.ColRequestedBy),
TaskTitle = taskTitle, TaskTitle = SafeGet(fields, _config.ColTask),
DeviceName = taskTitle, DeviceName = SafeGet(fields, _config.ColTask),
Location = SafeGet(fields, _config.ColLocation), Location = SafeGet(fields, _config.ColLocation),
IssueDescription = issueDesc ?? taskTitle, IssueDescription = SafeGet(fields, _config.ColIssue) ?? SafeGet(fields, _config.ColTask),
RawPriority = SafeGet(fields, _config.ColPriority), RawPriority = SafeGet(fields, _config.ColPriority),
RawStatus = SafeGet(fields, _config.ColStatus), RawStatus = SafeGet(fields, _config.ColStatus),
AssignedTo = SafeGet(fields, _config.ColAssignedTo), AssignedTo = SafeGet(fields, _config.ColAssignedTo),
@@ -145,8 +113,8 @@ namespace Disco.Plugins.ServiceTracker.Services
SheetNotes = SafeGet(fields, _config.ColNotes) SheetNotes = SafeGet(fields, _config.ColNotes)
}; };
// Generate STABLE IDs from content - same row always gets same ID // Hash uses RAW strings - never changes regardless of date parsing
var stableHash = GenerateStableHash(timestamp, email, taskTitle); var stableHash = GenerateStableHash(rawTimestamp, rawEmail, rawTask, rawIssue);
ticket.InternalId = StableIdFromHash(stableHash, usedIds); ticket.InternalId = StableIdFromHash(stableHash, usedIds);
ticket.ExternalId = "NTT" + ticket.InternalId.ToString(); ticket.ExternalId = "NTT" + ticket.InternalId.ToString();
usedIds.Add(ticket.InternalId); usedIds.Add(ticket.InternalId);
@@ -155,42 +123,43 @@ namespace Disco.Plugins.ServiceTracker.Services
ticket.IsOpen = status != "completed" && status != "closed" ticket.IsOpen = status != "completed" && status != "closed"
&& status != "resolved" && status != "done" && status != "cancelled"; && status != "resolved" && status != "done" && status != "cancelled";
if (ticket.IsOpen && (!string.IsNullOrWhiteSpace(taskTitle) || !string.IsNullOrWhiteSpace(issueDesc))) if (ticket.IsOpen && (!string.IsNullOrWhiteSpace(ticket.TaskTitle) || !string.IsNullOrWhiteSpace(ticket.IssueDescription)))
tickets.Add(ticket); tickets.Add(ticket);
} }
return tickets; return tickets;
} }
/// <summary> /// <summary>
/// Generates a deterministic hash from the row's key fields. /// Hash from RAW cell text - same spreadsheet cells always produce the same number.
/// Same timestamp + email + task always produces the same number. /// Uses raw strings, never parsed values, so DateTime.Now never contaminates the hash.
/// </summary> /// </summary>
private int GenerateStableHash(DateTime timestamp, string email, string task) private int GenerateStableHash(string rawTimestamp, string rawEmail, string rawTask, string rawIssue)
{ {
var key = timestamp.ToString("yyyyMMddHHmmss") + "|" var key = (rawTimestamp ?? "").Trim().ToLower() + "|"
+ (email ?? "").ToLower().Trim() + "|" + (rawEmail ?? "").Trim().ToLower() + "|"
+ (task ?? "").ToLower().Trim(); + (rawTask ?? "").Trim().ToLower() + "|"
+ (rawIssue ?? "").Trim().ToLower();
// Use a simple but stable hash (DJB2-like)
uint hash = 5381; uint hash = 5381;
for (int i = 0; i < key.Length; i++) for (int i = 0; i < key.Length; i++)
hash = ((hash << 5) + hash) + (uint)key[i]; hash = ((hash << 5) + hash) + (uint)key[i];
return (int)(hash % 800000) + 100000;
return (int)(hash % 800000) + 100000; // Range: 100000-899999
} }
/// <summary>
/// Returns the hash as an ID, resolving collisions by incrementing.
/// </summary>
private int StableIdFromHash(int baseId, HashSet<int> used) private int StableIdFromHash(int baseId, HashSet<int> used)
{ {
int id = baseId; int id = baseId;
while (used.Contains(id)) while (used.Contains(id)) id++;
id++;
return id; return id;
} }
/// <summary>Returns raw untrimmed cell value for hashing. Never returns null - returns empty string.</summary>
private string RawGet(List<string> fields, int index)
{
if (index < 0 || index >= fields.Count) return "";
return fields[index] ?? "";
}
public static string MapPriority(string raw) public static string MapPriority(string raw)
{ {
if (string.IsNullOrEmpty(raw)) return "medium"; if (string.IsNullOrEmpty(raw)) return "medium";
@@ -200,11 +169,6 @@ namespace Disco.Plugins.ServiceTracker.Services
if (lower.Contains("medium") || lower.Contains("3-5 day")) return "medium"; if (lower.Contains("medium") || lower.Contains("3-5 day")) return "medium";
if (lower.Contains("low") || lower.Contains("1 week") || lower.Contains("week+")) return "low"; if (lower.Contains("low") || lower.Contains("1 week") || lower.Contains("week+")) return "low";
if (lower.Contains("scheduled") || lower.Contains("planned")) return "scheduled"; if (lower.Contains("scheduled") || lower.Contains("planned")) return "scheduled";
// Emoji fallback
if (raw.Contains("\U0001F534")) return "critical";
if (raw.Contains("\U0001F7E0")) return "high";
if (raw.Contains("\U0001F7E1")) return "medium";
if (raw.Contains("\U0001F7E2")) return "low";
return "medium"; return "medium";
} }
@@ -218,10 +182,10 @@ namespace Disco.Plugins.ServiceTracker.Services
private DateTime SafeGetDate(List<string> fields, int index) private DateTime SafeGetDate(List<string> fields, int index)
{ {
var val = SafeGet(fields, index); var val = SafeGet(fields, index);
if (val == null) return DateTime.Now; if (val == null) return DateTime.MinValue;
DateTime dt; DateTime dt;
if (DateTime.TryParse(val, out dt)) return dt; if (DateTime.TryParse(val, out dt)) return dt;
return DateTime.Now; return DateTime.MinValue;
} }
private DateTime? SafeGetDateNullable(List<string> fields, int index) private DateTime? SafeGetDateNullable(List<string> fields, int index)