fix: hash RAW cell text for stable IDs, add rawIssue to hash, use DateTime.MinValue not DateTime.Now for unparseable dates
This commit is contained in:
@@ -28,7 +28,6 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
}
|
}
|
||||||
|
|
||||||
string csvData = null;
|
string csvData = null;
|
||||||
|
|
||||||
if (File.Exists(_cachePath))
|
if (File.Exists(_cachePath))
|
||||||
{
|
{
|
||||||
var cacheAge = DateTime.Now - File.GetLastWriteTime(_cachePath);
|
var cacheAge = DateTime.Now - File.GetLastWriteTime(_cachePath);
|
||||||
@@ -46,50 +45,25 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
url = "https://docs.google.com/spreadsheets/d/e/" + id + "/pub?output=csv&gid=" + _config.GId;
|
url = "https://docs.google.com/spreadsheets/d/e/" + id + "/pub?output=csv&gid=" + _config.GId;
|
||||||
else
|
else
|
||||||
url = "https://docs.google.com/spreadsheets/d/" + id + "/export?format=csv&gid=" + _config.GId;
|
url = "https://docs.google.com/spreadsheets/d/" + id + "/export?format=csv&gid=" + _config.GId;
|
||||||
|
|
||||||
using (var client = new WebClient())
|
using (var client = new WebClient())
|
||||||
{
|
{
|
||||||
client.Encoding = Encoding.UTF8;
|
client.Encoding = Encoding.UTF8;
|
||||||
client.Headers.Add("User-Agent", "Mozilla/5.0 DiscoServiceTracker/1.0");
|
client.Headers.Add("User-Agent", "Mozilla/5.0 DiscoServiceTracker/1.0");
|
||||||
csvData = client.DownloadString(url);
|
csvData = client.DownloadString(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (csvData != null && csvData.TrimStart().StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase))
|
if (csvData != null && csvData.TrimStart().StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
if (File.Exists(_cachePath))
|
if (File.Exists(_cachePath)) { csvData = File.ReadAllText(_cachePath); result.Warning = "Got HTML not CSV. Using cache."; }
|
||||||
{
|
else { result.Error = "Google returned HTML not CSV. Publish as CSV."; return result; }
|
||||||
csvData = File.ReadAllText(_cachePath);
|
|
||||||
result.Warning = "Got HTML instead of CSV. Using cache.";
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
result.Error = "Google returned HTML not CSV. Publish as CSV.";
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
File.WriteAllText(_cachePath, csvData);
|
|
||||||
}
|
}
|
||||||
|
else File.WriteAllText(_cachePath, csvData);
|
||||||
}
|
}
|
||||||
catch (WebException ex)
|
catch (WebException ex)
|
||||||
{
|
{
|
||||||
if (File.Exists(_cachePath))
|
if (File.Exists(_cachePath)) { csvData = File.ReadAllText(_cachePath); result.Warning = "Using cache. " + ex.Message; }
|
||||||
{
|
else { result.Error = "Fetch failed: " + ex.Message; return result; }
|
||||||
csvData = File.ReadAllText(_cachePath);
|
|
||||||
result.Warning = "Using cache. Error: " + ex.Message;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
result.Error = "Fetch failed: " + ex.Message;
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
result.Error = "Error: " + ex.Message;
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
catch (Exception ex) { result.Error = "Error: " + ex.Message; return result; }
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
@@ -98,11 +72,7 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
if (result.Tickets.Count == 0 && string.IsNullOrEmpty(result.Warning))
|
if (result.Tickets.Count == 0 && string.IsNullOrEmpty(result.Warning))
|
||||||
result.Warning = "Sheet OK but no open tickets found.";
|
result.Warning = "Sheet OK but no open tickets found.";
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex) { result.Error = "Parse error: " + ex.Message; }
|
||||||
{
|
|
||||||
result.Error = "Parse error: " + ex.Message;
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,28 +86,26 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
{
|
{
|
||||||
var fields = ParseCsvLine(lines[i]);
|
var fields = ParseCsvLine(lines[i]);
|
||||||
if (fields.Count == 0) continue;
|
if (fields.Count == 0) continue;
|
||||||
|
|
||||||
bool allEmpty = true;
|
bool allEmpty = true;
|
||||||
for (int j = 0; j < fields.Count; j++)
|
for (int j = 0; j < fields.Count; j++) if (!string.IsNullOrWhiteSpace(fields[j])) { allEmpty = false; break; }
|
||||||
if (!string.IsNullOrWhiteSpace(fields[j])) { allEmpty = false; break; }
|
|
||||||
if (allEmpty) continue;
|
if (allEmpty) continue;
|
||||||
|
|
||||||
var taskTitle = SafeGet(fields, _config.ColTask);
|
// Get RAW cell values for stable hashing (before any parsing)
|
||||||
var issueDesc = SafeGet(fields, _config.ColIssue);
|
var rawTimestamp = RawGet(fields, _config.ColTimestamp);
|
||||||
var requestedBy = SafeGet(fields, _config.ColRequestedBy);
|
var rawEmail = RawGet(fields, _config.ColEmail);
|
||||||
var email = SafeGet(fields, _config.ColEmail);
|
var rawTask = RawGet(fields, _config.ColTask);
|
||||||
var timestamp = SafeGetDate(fields, _config.ColTimestamp);
|
var rawIssue = RawGet(fields, _config.ColIssue);
|
||||||
|
|
||||||
var ticket = new ExternalTicket
|
var ticket = new ExternalTicket
|
||||||
{
|
{
|
||||||
Source = "ntt",
|
Source = "ntt",
|
||||||
Timestamp = timestamp,
|
Timestamp = SafeGetDate(fields, _config.ColTimestamp),
|
||||||
RequesterEmail = email,
|
RequesterEmail = SafeGet(fields, _config.ColEmail),
|
||||||
RequesterName = requestedBy,
|
RequesterName = SafeGet(fields, _config.ColRequestedBy),
|
||||||
TaskTitle = taskTitle,
|
TaskTitle = SafeGet(fields, _config.ColTask),
|
||||||
DeviceName = taskTitle,
|
DeviceName = SafeGet(fields, _config.ColTask),
|
||||||
Location = SafeGet(fields, _config.ColLocation),
|
Location = SafeGet(fields, _config.ColLocation),
|
||||||
IssueDescription = issueDesc ?? taskTitle,
|
IssueDescription = SafeGet(fields, _config.ColIssue) ?? SafeGet(fields, _config.ColTask),
|
||||||
RawPriority = SafeGet(fields, _config.ColPriority),
|
RawPriority = SafeGet(fields, _config.ColPriority),
|
||||||
RawStatus = SafeGet(fields, _config.ColStatus),
|
RawStatus = SafeGet(fields, _config.ColStatus),
|
||||||
AssignedTo = SafeGet(fields, _config.ColAssignedTo),
|
AssignedTo = SafeGet(fields, _config.ColAssignedTo),
|
||||||
@@ -145,8 +113,8 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
SheetNotes = SafeGet(fields, _config.ColNotes)
|
SheetNotes = SafeGet(fields, _config.ColNotes)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Generate STABLE IDs from content - same row always gets same ID
|
// Hash uses RAW strings - never changes regardless of date parsing
|
||||||
var stableHash = GenerateStableHash(timestamp, email, taskTitle);
|
var stableHash = GenerateStableHash(rawTimestamp, rawEmail, rawTask, rawIssue);
|
||||||
ticket.InternalId = StableIdFromHash(stableHash, usedIds);
|
ticket.InternalId = StableIdFromHash(stableHash, usedIds);
|
||||||
ticket.ExternalId = "NTT" + ticket.InternalId.ToString();
|
ticket.ExternalId = "NTT" + ticket.InternalId.ToString();
|
||||||
usedIds.Add(ticket.InternalId);
|
usedIds.Add(ticket.InternalId);
|
||||||
@@ -155,42 +123,43 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
ticket.IsOpen = status != "completed" && status != "closed"
|
ticket.IsOpen = status != "completed" && status != "closed"
|
||||||
&& status != "resolved" && status != "done" && status != "cancelled";
|
&& status != "resolved" && status != "done" && status != "cancelled";
|
||||||
|
|
||||||
if (ticket.IsOpen && (!string.IsNullOrWhiteSpace(taskTitle) || !string.IsNullOrWhiteSpace(issueDesc)))
|
if (ticket.IsOpen && (!string.IsNullOrWhiteSpace(ticket.TaskTitle) || !string.IsNullOrWhiteSpace(ticket.IssueDescription)))
|
||||||
tickets.Add(ticket);
|
tickets.Add(ticket);
|
||||||
}
|
}
|
||||||
|
|
||||||
return tickets;
|
return tickets;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Generates a deterministic hash from the row's key fields.
|
/// Hash from RAW cell text - same spreadsheet cells always produce the same number.
|
||||||
/// Same timestamp + email + task always produces the same number.
|
/// Uses raw strings, never parsed values, so DateTime.Now never contaminates the hash.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private int GenerateStableHash(DateTime timestamp, string email, string task)
|
private int GenerateStableHash(string rawTimestamp, string rawEmail, string rawTask, string rawIssue)
|
||||||
{
|
{
|
||||||
var key = timestamp.ToString("yyyyMMddHHmmss") + "|"
|
var key = (rawTimestamp ?? "").Trim().ToLower() + "|"
|
||||||
+ (email ?? "").ToLower().Trim() + "|"
|
+ (rawEmail ?? "").Trim().ToLower() + "|"
|
||||||
+ (task ?? "").ToLower().Trim();
|
+ (rawTask ?? "").Trim().ToLower() + "|"
|
||||||
|
+ (rawIssue ?? "").Trim().ToLower();
|
||||||
|
|
||||||
// Use a simple but stable hash (DJB2-like)
|
|
||||||
uint hash = 5381;
|
uint hash = 5381;
|
||||||
for (int i = 0; i < key.Length; i++)
|
for (int i = 0; i < key.Length; i++)
|
||||||
hash = ((hash << 5) + hash) + (uint)key[i];
|
hash = ((hash << 5) + hash) + (uint)key[i];
|
||||||
|
return (int)(hash % 800000) + 100000;
|
||||||
return (int)(hash % 800000) + 100000; // Range: 100000-899999
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Returns the hash as an ID, resolving collisions by incrementing.
|
|
||||||
/// </summary>
|
|
||||||
private int StableIdFromHash(int baseId, HashSet<int> used)
|
private int StableIdFromHash(int baseId, HashSet<int> used)
|
||||||
{
|
{
|
||||||
int id = baseId;
|
int id = baseId;
|
||||||
while (used.Contains(id))
|
while (used.Contains(id)) id++;
|
||||||
id++;
|
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Returns raw untrimmed cell value for hashing. Never returns null - returns empty string.</summary>
|
||||||
|
private string RawGet(List<string> fields, int index)
|
||||||
|
{
|
||||||
|
if (index < 0 || index >= fields.Count) return "";
|
||||||
|
return fields[index] ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
public static string MapPriority(string raw)
|
public static string MapPriority(string raw)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrEmpty(raw)) return "medium";
|
if (string.IsNullOrEmpty(raw)) return "medium";
|
||||||
@@ -200,11 +169,6 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
if (lower.Contains("medium") || lower.Contains("3-5 day")) return "medium";
|
if (lower.Contains("medium") || lower.Contains("3-5 day")) return "medium";
|
||||||
if (lower.Contains("low") || lower.Contains("1 week") || lower.Contains("week+")) return "low";
|
if (lower.Contains("low") || lower.Contains("1 week") || lower.Contains("week+")) return "low";
|
||||||
if (lower.Contains("scheduled") || lower.Contains("planned")) return "scheduled";
|
if (lower.Contains("scheduled") || lower.Contains("planned")) return "scheduled";
|
||||||
// Emoji fallback
|
|
||||||
if (raw.Contains("\U0001F534")) return "critical";
|
|
||||||
if (raw.Contains("\U0001F7E0")) return "high";
|
|
||||||
if (raw.Contains("\U0001F7E1")) return "medium";
|
|
||||||
if (raw.Contains("\U0001F7E2")) return "low";
|
|
||||||
return "medium";
|
return "medium";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,10 +182,10 @@ namespace Disco.Plugins.ServiceTracker.Services
|
|||||||
private DateTime SafeGetDate(List<string> fields, int index)
|
private DateTime SafeGetDate(List<string> fields, int index)
|
||||||
{
|
{
|
||||||
var val = SafeGet(fields, index);
|
var val = SafeGet(fields, index);
|
||||||
if (val == null) return DateTime.Now;
|
if (val == null) return DateTime.MinValue;
|
||||||
DateTime dt;
|
DateTime dt;
|
||||||
if (DateTime.TryParse(val, out dt)) return dt;
|
if (DateTime.TryParse(val, out dt)) return dt;
|
||||||
return DateTime.Now;
|
return DateTime.MinValue;
|
||||||
}
|
}
|
||||||
|
|
||||||
private DateTime? SafeGetDateNullable(List<string> fields, int index)
|
private DateTime? SafeGetDateNullable(List<string> fields, int index)
|
||||||
|
|||||||
Reference in New Issue
Block a user