|
|
using Microsoft.Azure.Cosmos; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks;
namespace CDP { internal class AuditDB { //https://learn.microsoft.com/en-us/azure/azure-functions/manage-connections?tabs=csharp#azure-cosmos-db-clients
private static Lazy<CosmosClient> lazyClient = new Lazy<CosmosClient>(InitializeCosmosClient); private static CosmosClient cosmosClient => lazyClient.Value; private static string DatabaseName = "CDP"; private static string FileAuditContainer = "FileAudits"; private static string UserAuditContainer = "UserAudits"; private static string GroupAuditContainer = "GroupAudits"; private static string TenantAuditContainer = "TenantAudits"; private static CosmosClient InitializeCosmosClient() { // Perform any initialization here
var uri = "https://cdplite.documents.azure.com:443/"; var authKey = "VPbg8RpzyI3XwhC2o0dIUtYFs33ghxORCqZeNAyg8vg4HWUBjM41BUxP0qLFXEvFh6ewQY1uKv52ACDbsEN1AQ==";
return new CosmosClient(uri, authKey); }
public static async Task UpdateAuditDocument(AuditDocument al, string containerName) { if (al.Records.Count == 0) return;
List<AuditDocument> lal = await SplitAuditlog(al);
Container container = cosmosClient.GetContainer(DatabaseName, containerName);
foreach (AuditDocument ial in lal) { ItemResponse<AuditDocument> r = await container.UpsertItemAsync(ial, new PartitionKey(ial.id)); }
await UpdateMetadata(container, lal); }
static async Task UpdateMetadata(Container container, List<AuditDocument> lal) { bool update = false; string pKey = GetMetaDocumentKey(lal[0].DocId); MetadataDocument md = null; try { ItemResponse<MetadataDocument> response = await container.ReadItemAsync<MetadataDocument>(pKey, new PartitionKey(pKey)); md = response.Resource; } catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) { md = new MetadataDocument() { id = lal[0].DocId + "-meta", PartitionKeys = new List<string>() }; } catch (Exception e) { // Helpers.LogIt(e.Message);
return; }
if (md == null) { // Helpers.LogIt("Something ugly happened!");
return; }
foreach (AuditDocument log in lal) { if (md.PartitionKeys.Contains(log.id)) continue; md.PartitionKeys.Add(log.id); update = true; }
if (update) { try { ItemResponse<MetadataDocument> r = await container.UpsertItemAsync(md, new PartitionKey(pKey)); } catch (Exception e) { // Helpers.LogIt(e.Message);
return; } } }
public static async Task<AuditDocument> GetAuditDocument(string key, string ContainerName) { try { Container container = cosmosClient.GetContainer(DatabaseName, ContainerName); ItemResponse<AuditDocument> response = await container.ReadItemAsync<AuditDocument>(key, new PartitionKey(key)); if (response == null) return null;
AuditDocument t = response.Resource; return t; } catch (Exception e) { return null; } }
static async Task<MetadataDocument> GetMetadataDocument(string id, string ContainerName) { MetadataDocument md = null; string pKey = GetMetaDocumentKey(id);
PartitionKey partitionKey = new PartitionKeyBuilder() .Add(id) .Build(); Container container = cosmosClient.GetContainer(DatabaseName, ContainerName); try { ItemResponse<MetadataDocument> response = await container.ReadItemAsync<MetadataDocument>(pKey, partitionKey); md = response.Resource; } catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) { md = new MetadataDocument() { id = id, PartitionKeys = new List<string>() }; } catch (Exception e) { // Helpers.LogIt(e.Message);
return null; }
if (md == null) { // Helpers.LogIt("Something ugly happened!");
return null; } return md; }
public static async Task<List<AuditRecord>> GetAuditRecordsBetweenDates(string DocId, DateTime startDate, DateTime endDate, string ContainerName) { var results = new List<AuditRecord>(); Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
// Fetch the metadata document for the customer ID
var metadataDocumentId = GetMetaDocumentKey(DocId); MetadataDocument metadataDocument = null; try { var metadataDocumentResponse = await container.ReadItemAsync<MetadataDocument>(metadataDocumentId, new PartitionKey(metadataDocumentId)); metadataDocument = metadataDocumentResponse.Resource; } catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) { } catch (Exception e) { // Helpers.LogIt(e.Message);
}
if (metadataDocument == null) return results;
// Determine the partition keys within the date range
var partitionKeysInRange = metadataDocument.PartitionKeys .Where(pk => IsPartitionKeyInRange(pk, startDate, endDate)) .ToList();
// Fetch the audit records for each partition key within the date range
foreach (var partitionKey in partitionKeysInRange) {
ItemResponse<AuditDocument> response = await container.ReadItemAsync<AuditDocument>(partitionKey, new PartitionKey(partitionKey)); if (response == null) continue;
AuditDocument t = response.Resource; results.AddRange(t.Records); }
return results; }
static private bool IsPartitionKeyInRange(string partitionKey, DateTime startDate, DateTime endDate) { var partitionKeyParts = partitionKey.Split('-'); var length = partitionKeyParts.Length;
var year = int.Parse(partitionKeyParts[length - 3]); var dayOfYear = int.Parse(partitionKeyParts[length - 2]); var partitionKeyDate = new DateTime(year, 1, 1).AddDays(dayOfYear - 1);
return partitionKeyDate >= startDate && partitionKeyDate <= endDate; }
public static async Task<bool> AppendRecord(string id, AuditRecord rec, string ContainerName) { try { var metadataDocument = await GetMetadataDocument(id, ContainerName); if (metadataDocument == null) return false;
string dayKey = metadataDocument.GetLatestKeyForDay(rec.EventTime); AuditDocument al = await GetAuditDocument(dayKey, ContainerName); if (al == null && ContainerName == FileAuditContainer) al = new AuditDocument() { DocId = rec.FileId }; else if (al == null && ContainerName == TenantAuditContainer) al = new AuditDocument() { DocId = rec.AppKey }; else if (al == null && ContainerName == UserAuditContainer) al = new AuditDocument() { DocId = rec.UserId }; else if (al == null && ContainerName == GroupAuditContainer) al = new AuditDocument() { DocId = rec.GroupId }; al.Records.Add(rec); await UpdateAuditDocument(al, ContainerName); return true; } catch (Exception e) { return false; } }
public static async Task<bool> AppendRecordsList(string id, List<AuditRecord> rec, string ContainerName) { try { var metadataDocument = await GetMetadataDocument(id, ContainerName); if (metadataDocument == null || rec.Count <= 0) return false;
string dayKey = metadataDocument.GetLatestKeyForDay(rec[0].EventTime); AuditDocument al = await GetAuditDocument(dayKey, ContainerName); /*if (al == null && ContainerName == FileAuditContainer) al = new AuditDocument() { DocId = rec.FileId };*/ if (al == null && ContainerName == TenantAuditContainer) al = new AuditDocument() { DocId = rec[0].AppKey }; else if (al == null && ContainerName == UserAuditContainer) al = new AuditDocument() { DocId = rec[0].UserId };
rec.ForEach(record => { al.Records.Add(record); }); await UpdateAuditDocument(al, ContainerName); return true; } catch (Exception e) { return false; } }
public static async Task<GroupAuditDocument> GetGroupAuditDocument(string appKey, string groupId) { Container container = cosmosClient.GetContainer(DatabaseName, GroupAuditContainer); // get the Group Container
GroupAuditDocument ad = new GroupAuditDocument() { GroupId = groupId }; try { // Store the unique identifier
string id = groupId;
PartitionKey partitionKey = new PartitionKeyBuilder() .Add(appKey) .Add(groupId) .Build();
// Perform a point read
ItemResponse<GroupAuditDocument> r = await container.ReadItemAsync<GroupAuditDocument>(id, partitionKey); return r; }
catch (Exception e) { return null; } } public static async Task AddTenantAuditRecord(FileAuditRecord far) { TenantAuditRecord uar = new TenantAuditRecord() { Action = far.Action, AppKey = far.AppKey, EventTime = far.EventTime, FileId = far.FileId, GroupId = far.GroupId, Message = far.Message, UserId = far.UserId };
// cosmosClient is init'd by the static on line 21
Container container = cosmosClient.GetContainer(DatabaseName, TenantAuditContainer); // Build the full partition key path
PartitionKey partitionKey = new PartitionKeyBuilder() .Add(uar.AppKey) .Build();
TenantAuditDocument ad = await GetTenantAuditDocument(uar.AppKey); if (ad == null) ad = new TenantAuditDocument() { AppKey = uar.AppKey };
ad.Records.Add(uar); try { ItemResponse<TenantAuditDocument> r = await container.UpsertItemAsync(ad, partitionKey); } catch (Exception e) { Console.WriteLine(e); } }
public static async Task<TenantAuditDocument> GetTenantAuditDocument(string appKey) { Container container = cosmosClient.GetContainer(DatabaseName, TenantAuditContainer); // get the Group Container
TenantAuditDocument ad = new TenantAuditDocument() { AppKey = appKey }; try { // Store the unique identifier
string id = appKey;
PartitionKey partitionKey = new PartitionKeyBuilder() .Add(appKey) .Build();
// Perform a point read
ItemResponse<TenantAuditDocument> r = await container.ReadItemAsync<TenantAuditDocument>(id, partitionKey); return r; }
catch (Exception e) { return null; } }
static async Task<List<AuditDocument>> SplitAuditlog(AuditDocument al) { List<AuditDocument> lal = new List<AuditDocument>(); var sortedRecords = al.Records.OrderBy(record => record.EventTime).ToList();
var currentGroup = new List<AuditRecord>(); var currentGroupSize = 0;
int MaxDocumentSizeInBytes = 2 * 1024 * 1024; // 2MB
int index = al.Index; // start for index passed in
foreach (var record in sortedRecords) { var recordSize = record.CalculateRecordSize();
if (currentGroupSize + recordSize > MaxDocumentSizeInBytes) { AuditDocument i = new AuditDocument(); i.Index = index++; i.Records = currentGroup; lal.Add(i);
currentGroup = new List<AuditRecord>(); currentGroupSize = 0; }
currentGroup.Add(record); currentGroupSize += recordSize; }
if (currentGroup.Any()) { AuditDocument i = new AuditDocument(); i.DocId = al.DocId; i.Index = index++; i.Records = currentGroup; lal.Add(i); }
return lal; }
static string GetMetaDocumentKey(string id) { return $"{id}-meta"; } } }
|