You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
404 lines
14 KiB
404 lines
14 KiB
using Microsoft.Azure.Cosmos;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace CDP
|
|
{
|
|
internal class AuditDB
|
|
{
|
|
//https://learn.microsoft.com/en-us/azure/azure-functions/manage-connections?tabs=csharp#azure-cosmos-db-clients
|
|
private static Lazy<CosmosClient> lazyClient = new Lazy<CosmosClient>(InitializeCosmosClient);
|
|
private static CosmosClient cosmosClient => lazyClient.Value;
|
|
private static string DatabaseName = "CDP";
|
|
private static string FileAuditContainer = "FileAudits";
|
|
private static string UserAuditContainer = "UserAudits";
|
|
private static string GroupAuditContainer = "GroupAudits";
|
|
private static string TenantAuditContainer = "TenantAudits";
|
|
private static CosmosClient InitializeCosmosClient()
|
|
{
|
|
// Perform any initialization here
|
|
var uri = "https://cdplite.documents.azure.com:443/";
|
|
var authKey = "VPbg8RpzyI3XwhC2o0dIUtYFs33ghxORCqZeNAyg8vg4HWUBjM41BUxP0qLFXEvFh6ewQY1uKv52ACDbsEN1AQ==";
|
|
|
|
return new CosmosClient(uri, authKey);
|
|
}
|
|
|
|
public static async Task UpdateAuditDocument(AuditDocument al, string containerName)
|
|
{
|
|
if (al.Records.Count == 0)
|
|
return;
|
|
|
|
List<AuditDocument> lal = await SplitAuditlog(al);
|
|
|
|
Container container = cosmosClient.GetContainer(DatabaseName, containerName);
|
|
|
|
foreach (AuditDocument ial in lal)
|
|
{
|
|
ItemResponse<AuditDocument> r = await container.UpsertItemAsync(ial, new PartitionKey(ial.id));
|
|
}
|
|
|
|
await UpdateMetadata(container, lal);
|
|
}
|
|
|
|
static async Task UpdateMetadata(Container container, List<AuditDocument> lal)
|
|
{
|
|
bool update = false;
|
|
string pKey = GetMetaDocumentKey(lal[0].DocId);
|
|
MetadataDocument md = null;
|
|
try
|
|
{
|
|
ItemResponse<MetadataDocument> response =
|
|
await container.ReadItemAsync<MetadataDocument>(pKey, new PartitionKey(pKey));
|
|
md = response.Resource;
|
|
}
|
|
catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
|
|
{
|
|
md = new MetadataDocument()
|
|
{
|
|
id = lal[0].DocId + "-meta",
|
|
PartitionKeys = new List<string>()
|
|
};
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
// Helpers.LogIt(e.Message);
|
|
return;
|
|
}
|
|
|
|
if (md == null)
|
|
{
|
|
// Helpers.LogIt("Something ugly happened!");
|
|
return;
|
|
}
|
|
|
|
foreach (AuditDocument log in lal)
|
|
{
|
|
if (md.PartitionKeys.Contains(log.id))
|
|
continue;
|
|
md.PartitionKeys.Add(log.id);
|
|
update = true;
|
|
}
|
|
|
|
if (update)
|
|
{
|
|
try
|
|
{
|
|
ItemResponse<MetadataDocument> r = await container.UpsertItemAsync(md, new PartitionKey(pKey));
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
// Helpers.LogIt(e.Message);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
public static async Task<AuditDocument> GetAuditDocument(string key, string ContainerName)
|
|
{
|
|
try
|
|
{
|
|
Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
|
|
ItemResponse<AuditDocument> response = await container.ReadItemAsync<AuditDocument>(key, new PartitionKey(key));
|
|
if (response == null)
|
|
return null;
|
|
|
|
AuditDocument t = response.Resource;
|
|
return t;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
static async Task<MetadataDocument> GetMetadataDocument(string id, string ContainerName)
|
|
{
|
|
MetadataDocument md = null;
|
|
string pKey = GetMetaDocumentKey(id);
|
|
|
|
PartitionKey partitionKey = new PartitionKeyBuilder()
|
|
.Add(id)
|
|
.Build();
|
|
Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
|
|
try
|
|
{
|
|
ItemResponse<MetadataDocument> response =
|
|
await container.ReadItemAsync<MetadataDocument>(pKey, partitionKey);
|
|
md = response.Resource;
|
|
}
|
|
catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
|
|
{
|
|
md = new MetadataDocument()
|
|
{
|
|
id = id,
|
|
PartitionKeys = new List<string>()
|
|
};
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
// Helpers.LogIt(e.Message);
|
|
return null;
|
|
}
|
|
|
|
if (md == null)
|
|
{
|
|
// Helpers.LogIt("Something ugly happened!");
|
|
return null;
|
|
}
|
|
return md;
|
|
}
|
|
|
|
public static async Task<List<AuditRecord>> GetAuditRecordsBetweenDates(string DocId, DateTime startDate, DateTime endDate, string ContainerName)
|
|
{
|
|
var results = new List<AuditRecord>();
|
|
Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
|
|
|
|
// Fetch the metadata document for the customer ID
|
|
var metadataDocumentId = GetMetaDocumentKey(DocId);
|
|
MetadataDocument metadataDocument = null;
|
|
try
|
|
{
|
|
var metadataDocumentResponse =
|
|
await container.ReadItemAsync<MetadataDocument>(metadataDocumentId, new PartitionKey(metadataDocumentId));
|
|
metadataDocument = metadataDocumentResponse.Resource;
|
|
}
|
|
catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
|
|
{
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
// Helpers.LogIt(e.Message);
|
|
}
|
|
|
|
if (metadataDocument == null)
|
|
return results;
|
|
|
|
// Determine the partition keys within the date range
|
|
var partitionKeysInRange = metadataDocument.PartitionKeys
|
|
.Where(pk => IsPartitionKeyInRange(pk, startDate, endDate))
|
|
.ToList();
|
|
|
|
// Fetch the audit records for each partition key within the date range
|
|
foreach (var partitionKey in partitionKeysInRange)
|
|
{
|
|
|
|
ItemResponse<AuditDocument> response = await container.ReadItemAsync<AuditDocument>(partitionKey, new PartitionKey(partitionKey));
|
|
if (response == null)
|
|
continue;
|
|
|
|
AuditDocument t = response.Resource;
|
|
results.AddRange(t.Records);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
static private bool IsPartitionKeyInRange(string partitionKey, DateTime startDate, DateTime endDate)
|
|
{
|
|
var partitionKeyParts = partitionKey.Split('-');
|
|
var length = partitionKeyParts.Length;
|
|
|
|
var year = int.Parse(partitionKeyParts[length - 3]);
|
|
var dayOfYear = int.Parse(partitionKeyParts[length - 2]);
|
|
var partitionKeyDate = new DateTime(year, 1, 1).AddDays(dayOfYear - 1);
|
|
|
|
return partitionKeyDate >= startDate && partitionKeyDate <= endDate;
|
|
}
|
|
|
|
public static async Task<bool> AppendRecord(string id, AuditRecord rec, string ContainerName)
|
|
{
|
|
try
|
|
{
|
|
var metadataDocument = await GetMetadataDocument(id, ContainerName);
|
|
if (metadataDocument == null)
|
|
return false;
|
|
|
|
string dayKey = metadataDocument.GetLatestKeyForDay(rec.EventTime);
|
|
AuditDocument al = await GetAuditDocument(dayKey, ContainerName);
|
|
if (al == null && ContainerName == FileAuditContainer)
|
|
al = new AuditDocument() { DocId = rec.FileId };
|
|
else if (al == null && ContainerName == TenantAuditContainer)
|
|
al = new AuditDocument() { DocId = rec.AppKey };
|
|
else if (al == null && ContainerName == UserAuditContainer)
|
|
al = new AuditDocument() { DocId = rec.UserId };
|
|
else if (al == null && ContainerName == GroupAuditContainer)
|
|
al = new AuditDocument() { DocId = rec.GroupId };
|
|
al.Records.Add(rec);
|
|
await UpdateAuditDocument(al, ContainerName);
|
|
return true;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public static async Task<bool> AppendRecordsList(string id, List<AuditRecord> rec, string ContainerName)
|
|
{
|
|
try
|
|
{
|
|
var metadataDocument = await GetMetadataDocument(id, ContainerName);
|
|
if (metadataDocument == null || rec.Count <= 0)
|
|
return false;
|
|
|
|
string dayKey = metadataDocument.GetLatestKeyForDay(rec[0].EventTime);
|
|
AuditDocument al = await GetAuditDocument(dayKey, ContainerName);
|
|
/*if (al == null && ContainerName == FileAuditContainer)
|
|
al = new AuditDocument() { DocId = rec.FileId };*/
|
|
if (al == null && ContainerName == TenantAuditContainer)
|
|
al = new AuditDocument() { DocId = rec[0].AppKey };
|
|
else if (al == null && ContainerName == UserAuditContainer)
|
|
al = new AuditDocument() { DocId = rec[0].UserId };
|
|
|
|
rec.ForEach(record =>
|
|
{
|
|
al.Records.Add(record);
|
|
});
|
|
await UpdateAuditDocument(al, ContainerName);
|
|
return true;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
public static async Task<GroupAuditDocument> GetGroupAuditDocument(string appKey, string groupId)
|
|
{
|
|
Container container = cosmosClient.GetContainer(DatabaseName, GroupAuditContainer); // get the Group Container
|
|
|
|
GroupAuditDocument ad = new GroupAuditDocument() { GroupId = groupId };
|
|
try
|
|
{
|
|
// Store the unique identifier
|
|
string id = groupId;
|
|
|
|
PartitionKey partitionKey = new PartitionKeyBuilder()
|
|
.Add(appKey)
|
|
.Add(groupId)
|
|
.Build();
|
|
|
|
// Perform a point read
|
|
ItemResponse<GroupAuditDocument> r = await container.ReadItemAsync<GroupAuditDocument>(id, partitionKey);
|
|
return r;
|
|
}
|
|
|
|
catch (Exception e)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
public static async Task AddTenantAuditRecord(FileAuditRecord far)
|
|
{
|
|
TenantAuditRecord uar = new TenantAuditRecord()
|
|
{
|
|
Action = far.Action,
|
|
AppKey = far.AppKey,
|
|
EventTime = far.EventTime,
|
|
FileId = far.FileId,
|
|
GroupId = far.GroupId,
|
|
Message = far.Message,
|
|
UserId = far.UserId
|
|
};
|
|
|
|
// cosmosClient is init'd by the static on line 21
|
|
Container container = cosmosClient.GetContainer(DatabaseName, TenantAuditContainer);
|
|
// Build the full partition key path
|
|
PartitionKey partitionKey = new PartitionKeyBuilder()
|
|
.Add(uar.AppKey)
|
|
.Build();
|
|
|
|
TenantAuditDocument ad = await GetTenantAuditDocument(uar.AppKey);
|
|
if (ad == null)
|
|
ad = new TenantAuditDocument() { AppKey = uar.AppKey };
|
|
|
|
ad.Records.Add(uar);
|
|
try
|
|
{
|
|
ItemResponse<TenantAuditDocument> r = await container.UpsertItemAsync(ad, partitionKey);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine(e);
|
|
}
|
|
}
|
|
|
|
public static async Task<TenantAuditDocument> GetTenantAuditDocument(string appKey)
|
|
{
|
|
Container container = cosmosClient.GetContainer(DatabaseName, TenantAuditContainer); // get the Group Container
|
|
|
|
TenantAuditDocument ad = new TenantAuditDocument() { AppKey = appKey };
|
|
try
|
|
{
|
|
// Store the unique identifier
|
|
string id = appKey;
|
|
|
|
PartitionKey partitionKey = new PartitionKeyBuilder()
|
|
.Add(appKey)
|
|
.Build();
|
|
|
|
// Perform a point read
|
|
ItemResponse<TenantAuditDocument> r = await container.ReadItemAsync<TenantAuditDocument>(id, partitionKey);
|
|
return r;
|
|
}
|
|
|
|
catch (Exception e)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
static async Task<List<AuditDocument>> SplitAuditlog(AuditDocument al)
|
|
{
|
|
List<AuditDocument> lal = new List<AuditDocument>();
|
|
var sortedRecords = al.Records.OrderBy(record => record.EventTime).ToList();
|
|
|
|
var currentGroup = new List<AuditRecord>();
|
|
var currentGroupSize = 0;
|
|
|
|
int MaxDocumentSizeInBytes = 2 * 1024 * 1024; // 2MB
|
|
|
|
int index = al.Index; // start for index passed in
|
|
foreach (var record in sortedRecords)
|
|
{
|
|
var recordSize = record.CalculateRecordSize();
|
|
|
|
if (currentGroupSize + recordSize > MaxDocumentSizeInBytes)
|
|
{
|
|
AuditDocument i = new AuditDocument();
|
|
i.Index = index++;
|
|
i.Records = currentGroup;
|
|
lal.Add(i);
|
|
|
|
currentGroup = new List<AuditRecord>();
|
|
currentGroupSize = 0;
|
|
}
|
|
|
|
currentGroup.Add(record);
|
|
currentGroupSize += recordSize;
|
|
}
|
|
|
|
if (currentGroup.Any())
|
|
{
|
|
AuditDocument i = new AuditDocument();
|
|
i.DocId = al.DocId;
|
|
i.Index = index++;
|
|
i.Records = currentGroup;
|
|
lal.Add(i);
|
|
}
|
|
|
|
return lal;
|
|
}
|
|
|
|
static string GetMetaDocumentKey(string id)
|
|
{
|
|
return $"{id}-meta";
|
|
}
|
|
}
|
|
}
|