CDP-LTS-80.sln

@@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.9.34622.214
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CDP-LTS-80", "CDP-LTS-80\CDP-LTS-80.csproj", "{C9E49B28-1B40-4C62-AF86-C674677D3AB9}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CDP-LTS-80", "CDP-LTS-80\CDP-LTS-80.csproj", "{C9E49B28-1B40-4C62-AF86-C674677D3AB9}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

CDP-LTS-80/Audit/AuditFunctions.cs

@@ -0,0 +1,263 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace CDP
+{
+    public class AuditFunctions
+    {
+        private readonly ILogger<AuditFunctions> _logger;
+        public static string FileAuditContainer = "FileAudits";
+        public static string UserAuditContainer = "UserAudits";
+        public static string GroupAuditContainer = "GroupAudits";
+        public static string TenantAuditContainer = "TenantAudits";
+        public AuditFunctions(ILogger<AuditFunctions> logger)
+        {
+            _logger = logger;
+        }
+
+        [Function("GetAuditLogForFile")]
+        public async Task<IActionResult> GetAuditLogForFile([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("GetAuditLogForFile invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            if (string.IsNullOrEmpty(requestBody))
+                return new BadRequestObjectResult(new { error = true, message = "The body is empty" });
+
+            _logger.LogInformation(requestBody);
+
+            GetAuditLogForFileDto dto = JsonConvert.DeserializeObject<GetAuditLogForFileDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            List<AuditRecord> ad = await AuditDB.GetAuditRecordsBetweenDates(dto.FileId, DateTime.MinValue, DateTime.MaxValue, CDPLite.FileAuditContainer);
+
+            return new OkObjectResult(ad);
+        }
+
+        [Function("GetAuditLogForUser")]
+        public async Task<IActionResult> GetAuditLogForUser([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("GetAuditLogForUser invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            if (string.IsNullOrEmpty(requestBody))
+                return new BadRequestObjectResult(new { error = true, message = "The body is empty" });
+
+            _logger.LogInformation(requestBody);
+
+            GetAuditLogForUserDto dto = JsonConvert.DeserializeObject<GetAuditLogForUserDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
+            List<AuditRecord> ad = await AuditDB.GetAuditRecordsBetweenDates(userId, DateTime.MinValue, DateTime.MaxValue, UserAuditContainer);
+            return new OkObjectResult(ad);
+        }
+
+        [Function("GetAuditLogForGroup")]
+        public async Task<IActionResult> GetAuditLogForGroup([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("GetAuditLogForGroup invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            if (string.IsNullOrEmpty(requestBody))
+                return new BadRequestObjectResult(new { error = true, message = "The body is empty" });
+
+            _logger.LogInformation(requestBody);
+
+            GetAuditLogForGroupDto dto = JsonConvert.DeserializeObject<GetAuditLogForGroupDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            List<AuditRecord> ad = await AuditDB.GetAuditRecordsBetweenDates(dto.GroupId, DateTime.MinValue, DateTime.MaxValue, GroupAuditContainer);
+            return new OkObjectResult(ad);
+        }
+
+        [Function("GetAuditLogForTenant")]
+        public async Task<IActionResult> GetAuditLogForTenant([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("GetAuditLogForGroup invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            if (string.IsNullOrEmpty(requestBody))
+                return new BadRequestObjectResult(new { error = true, message = "The body is empty" });
+
+            _logger.LogInformation(requestBody);
+
+            GetAuditLogForTenantDto dto = JsonConvert.DeserializeObject<GetAuditLogForTenantDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            List<AuditRecord> ad = await AuditDB.GetAuditRecordsBetweenDates(dto.AppKey, DateTime.MinValue, DateTime.MaxValue, TenantAuditContainer);
+            return new OkObjectResult(ad);
+        }
+
+        [Function("AddAccessViolation")]
+        public async Task<Boolean> AddAccessViolation([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("AddAccessViolation invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            if (string.IsNullOrEmpty(requestBody))
+                return false;
+
+            _logger.LogInformation(requestBody);
+
+            AddAccessViolationDto dto = JsonConvert.DeserializeObject<AddAccessViolationDto>(requestBody);
+            if (dto == null)
+                return false;
+
+            string message = string.Format($"Access violation recorded for file {dto.FileName}");
+            string action = "Access Violation";
+            await AddAudits(dto.AppKey, dto.FileId, dto.FileName, "", "", action, message);
+            return true;
+        }
+
+        public static async Task<string> AddAuditsEvent(string appKey, string fileId, string fileName, string userId, string groupid, string action, string message)
+        {
+            using (var mt = new MethodTimer("AddAuditsEventMessage"))
+            {
+                if (string.IsNullOrEmpty(appKey) || string.IsNullOrEmpty(fileId) || string.IsNullOrEmpty(action) || string.IsNullOrEmpty(message))
+                    return "";
+                AuditEventMetadata auditEvent = new AuditEventMetadata
+                {
+                    FileId = fileId,
+                    FileName = fileName,
+                    UserId = userId,
+                    GroupId = groupid,
+                    Action = action,
+                    Message = message
+                };
+                string jobId = Guid.NewGuid().ToString();
+                string jobMeta = JsonConvert.SerializeObject(auditEvent);
+                Job job = new Job { AppKey = appKey, EventType = JobType.AddAudits, Id = jobId, JobMetadata = jobMeta };
+                await MetaProcessor.PublishJob(job);
+                return jobId;
+            }
+        }
+
+
+        public static async Task AddAudits(string appKey, string fileId, string fileName, string userId, string groupid, string action, string message)
+        {
+            if (string.IsNullOrEmpty(appKey) || string.IsNullOrEmpty(fileId) || string.IsNullOrEmpty(action) || string.IsNullOrEmpty(message))
+            {
+                Console.WriteLine(string.Format("something weird? appKey, fileId, action, message: {0} {1} {2} {3}", appKey, fileId, action, message));
+                return;
+            }
+
+
+            AuditRecord faRec = new FileAuditRecord()
+            {
+                AppKey = appKey,
+                FileId = fileId,
+                FileName = fileName,
+                UserId = userId,
+                GroupId = groupid,
+                Action = action,
+                Message = message,
+                EventTime = DateTime.UtcNow,
+            };
+            Console.WriteLine("Adding File Audit Record");
+            await AuditDB.AppendRecord(faRec.id, faRec, FileAuditContainer);
+
+            AuditRecord faRecTenant = new TenantAuditRecord()
+            {
+                AppKey = appKey,
+                FileId = fileId,
+                FileName = fileName,
+                UserId = userId,
+                GroupId = groupid,
+                Action = action,
+                Message = message,
+                EventTime = DateTime.UtcNow,
+            };
+
+            await AuditDB.AppendRecord(faRecTenant.id, faRecTenant, TenantAuditContainer);
+
+            if (!string.IsNullOrEmpty(groupid))
+            {
+                AuditRecord faRecGroup = new GroupAuditRecord()
+                {
+                    AppKey = appKey,
+                    FileId = fileId,
+                    FileName = fileName,
+                    UserId = userId,
+                    GroupId = groupid,
+                    Action = action,
+                    Message = message,
+                    EventTime = DateTime.UtcNow,
+                };
+
+                await AuditDB.AppendRecord(faRecGroup.id, faRecGroup, GroupAuditContainer);
+            }
+
+            AuditRecord faRecUser = new UserAuditRecord()
+            {
+                AppKey = appKey,
+                FileId = fileId,
+                FileName = fileName,
+                UserId = userId,
+                GroupId = groupid,
+                Action = action,
+                Message = message,
+                EventTime = DateTime.UtcNow,
+            };
+
+            await AuditDB.AppendRecord(faRecUser.id, faRecUser, UserAuditContainer);
+
+        }
+        /// <summary>
+        ///  Adds the audit record on a background thread.
+        /// </summary>
+        private static async Task AddFileAudit(AuditRecord far)
+        {
+            await AuditDB.AppendRecord(far.id, far, FileAuditContainer);
+
+        }
+
+        private static async Task AddUserAudit(AuditRecord far)
+        {
+            await AuditDB.AppendRecord(far.id, far, UserAuditContainer);
+        }
+
+        private static async Task AddTenantAudit(AuditRecord far)
+        {
+            await Task.Run(async () =>
+            {
+                try
+                {
+                    await AuditDB.AppendRecord(far.id, far, TenantAuditContainer);
+                }
+                catch (Exception e)
+                {
+                }
+            });
+
+        }
+
+        private static async Task AddGroupAudit(AuditRecord far)
+        {
+            await Task.Run(async () =>
+            {
+                try
+                {
+                    await AuditDB.AppendRecord(far.id, far, GroupAuditContainer);
+                }
+                catch (Exception e)
+                {
+                }
+            });
+
+        }
+    }
+}

CDP-LTS-80/CDP-LTS-80.csproj

@@ -6,19 +6,25 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <RootNamespace>CDP</RootNamespace>
+    <UserSecretsId>efb0fbf7-afd0-468d-9e4c-605bbab91059</UserSecretsId>
   </PropertyGroup>
+  <ItemGroup>
+    <Content Include="local.settings.json" />
+  </ItemGroup>
   <ItemGroup>
     <FrameworkReference Include="Microsoft.AspNetCore.App" />
     <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.39.1" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.22.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.1.1" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.1.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="1.2.1" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.17.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.2" />
     <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.22.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="1.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

CDP-LTS-80/CDPCore/CDPLite.cs

@@ -0,0 +1,344 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Cosmos;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CDP
+{
+    public class CDPLite
+    {
+        private readonly ILogger<CDPLite> _logger;
+        public static string FileAuditContainer = "FileAudits";
+        public static string UserAuditContainer = "UserAudits";
+        public static string GroupAuditContainer = "GroupAudits";
+        public static string TenantAuditContainer = "TenantAudits";
+
+        public CDPLite(ILogger<CDPLite> log)
+        {
+            _logger = log;
+        }
+
+        /*internal async Task<IActionResult> AddFilesBatchedInternal(AddFileBatchedDto dto)
+        {
+            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
+            string jobId = Guid.NewGuid().ToString();
+            List<KeyVaultEvent> vaultEvents = new List<KeyVaultEvent>();
+            //List<AuditEventMetadata> auditEvents = new List<AuditEventMetadata>();
+            List<FileRecord> fileRecords = new List<FileRecord>();
+            for (int i = 0; i < dto.Count; i++)
+            {
+                string fileId = Guid.NewGuid().ToString();
+                string fileName = dto.FileNames[i];
+                string aesKey = Helpers.GenerateAES256KeyToBase64();
+                //string message = string.Format($"{dto.Email} protected {fileName} file having {fileId} id.");
+                KeyVaultEvent vaultEvent = new KeyVaultEvent { AESKey = aesKey, FileId = fileId };
+                //AuditEventMetadata auditEvent = new AuditEventMetadata { Action = "Addded", FileId = fileId, FileName = dto.FileNames[i], Message = message, UserId = userId };
+                //auditEvents.Add(auditEvent);
+                vaultEvents.Add(vaultEvent);
+
+                AccessPolicy ac = new AccessPolicy()
+                {
+                    Access = "Owner",
+                    Key = aesKey,
+                    Email = dto.Email
+                };
+                FileRecord fr = await CDPDB.UpsertFile(dto.AppKey, fileId, fileName, userId, "", ac);
+                fileRecords.Add(fr);
+            }
+            List<Job> jobs = new List<Job>();
+            Job vaultJob = await AddKeyVaultBatchedEvent(vaultEvents, dto.AppKey);
+            //Job auditJob = await AddAuditsBatchedEvent(auditEvents, dto.AppKey);
+            //jobs.Add(auditJob);
+            jobs.Add(vaultJob);
+            await MetaProcessor.PublishBatchJobs(jobs);
+            return new OkObjectResult(fileRecords);
+
+        }*/
+
+
+        internal async Task<IActionResult> AddFileInternal(AddFileDto dto, bool useKeyVaultEvent = false)
+        {
+            string fileId = Guid.NewGuid().ToString();
+
+            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
+            string fileName = dto.FileName;
+            string aesKey = Helpers.GenerateAES256KeyToBase64();
+
+            // the KeyVault is slow for some reason and while it's dangerous to return a key
+            // before we're sure it got added to the database...I'm going to do it anyway.
+            if (useKeyVaultEvent)
+            {
+                await AddKeyVaultEvent(fileId, aesKey, dto.AppKey);
+            }
+            else
+            {
+                await Task.Run(async () =>
+                {
+                    try
+                    {
+                        KeyVaultService kvs = new KeyVaultService(Constants.KeyVaultURI);
+                        await kvs.SetSecretAsync(fileId, aesKey);
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine(e);
+                    }
+                });
+            }
+
+            // when you add a file, you have rights to manage access to it.
+            AccessPolicy ac = new AccessPolicy()
+            {
+                Access = "Owner",
+                Key = aesKey,
+                Email = dto.Email
+            };
+
+            // since we're generating a new file id, a new entry will always be created.
+            FileRecord fr = await CDPDB.UpsertFile(dto.AppKey, fileId, fileName, userId, "", ac);
+
+            string message = string.Format($"{dto.Email} protected {fileName} file having {fileId} id.");
+            string action = "Added";
+
+            await AuditFunctions.AddAuditsEvent(dto.AppKey, fileId, fileName, userId, dto.GroupId, action, message); //commenting for speed test
+            return new OkObjectResult(fr);
+        }
+
+        internal static async Task<IActionResult> AddFileUserInternal(AddFileUserDto dto)
+        {
+            // check to see if the email has the power to add a user
+            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
+
+            FileRecord fr = await CDPDB.GetFile(dto.AppKey, dto.FileId, userId);
+            if (fr == null)
+            {
+                string message = string.Format($"{dto.Email} attempted to add/change access policy for {dto.EmailToAdd} on {dto.FileName} file having {dto.FileId} id, but didn't have ANY access");
+                Console.WriteLine(message);
+                string action = "Policy change failed";
+                await AuditFunctions.AddAuditsEvent(dto.AppKey, dto.FileId, dto.FileName, userId, "", action, message);
+
+                return new BadRequestObjectResult(new { error = true, message = "File not found for user " + dto.Email });
+            }
+
+            if ((!fr.Policy.CheckAccess("Manage")) && (!fr.Policy.CheckAccess("Owner")))
+            {
+                string message = string.Format($"{dto.Email} attempted to add/change access policy for {dto.EmailToAdd} on {dto.FileName} file having {dto.FileId} id, but didn't have manage access");
+                Console.WriteLine(message);
+                string action = "Policy change failed";
+                await AuditFunctions.AddAuditsEvent(dto.AppKey, dto.FileId, dto.FileName, userId, "", action, message);
+                return new BadRequestObjectResult(new { error = true, message = $"{dto.Email} doesn't have the rights to add a user." });
+            }
+
+            string fileId = dto.FileId;
+            string fileName = dto.FileName;
+            string userIdToAdd = "";
+
+            if (dto.EmailToAdd != "")
+            {
+                userIdToAdd = Helpers.HashAndShortenText(dto.EmailToAdd.ToLower());
+            }
+            else if (dto.Group != null)
+            {
+                userIdToAdd = dto.GroupId;
+            }
+            else if (dto.Group != null)
+            {
+                userIdToAdd = dto.GroupId;
+            }
+
+            AccessPolicy ac = new AccessPolicy()
+            {
+                Access = dto.Policy,
+                Email = dto.EmailToAdd.ToLower(),
+                Group = dto.Group,
+                GroupId = dto.GroupId,
+                Key = ""
+            };
+
+            fr = await CDPDB.UpsertFile(dto.AppKey, fileId, fileName, userIdToAdd, "", ac);
+
+            if (dto.EmailToAdd != "")
+            {
+                string message = string.Format($"{dto.Email} added/changed the access policy for User : {dto.EmailToAdd} to {dto.Policy} on {fileName} file having {fileId} id");
+                string action = "Policy change";
+                await AuditFunctions.AddAuditsEvent(dto.AppKey, fileId, fileName, userId, "", action, message);
+            }
+
+            if (dto.Group != null)
+            {
+                string message = string.Format($"{dto.Email} added/changed the access policy for Group : {dto.Group} to {dto.Policy} on {fileName} file having {fileId} id");
+                string action = "Policy change";
+                await AuditFunctions.AddAuditsEvent(dto.AppKey, fileId, fileName, "", dto.Group.id, action, message);
+            }
+            return new OkObjectResult(fr);
+        }
+
+        #region CDP File Functions
+
+        [Function("AddFile")]
+        public async Task<IActionResult> AddFile([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("AddFile invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            AddFileDto dto = JsonConvert.DeserializeObject<AddFileDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            return await AddFileInternal(dto, true);
+        }
+
+        [Function("AddFileUser")]
+        public async Task<IActionResult> AddFileUser([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("AddFileUser invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            AddFileUserDto dto = JsonConvert.DeserializeObject<AddFileUserDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            return await AddFileUserInternal(dto);
+        }
+
+        [Function("GetFileForUser")]
+        public async Task<IActionResult> GetFileForUser([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("GetFile invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            AddFileUserDto dto = JsonConvert.DeserializeObject<AddFileUserDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            // check to see if the email has the power to add a user
+            // string a = Helpers.HashAndShortenText(dto.Email.ToLower());
+            // string b = Helpers.HashToHex(dto.Email.ToLower());
+            // string c = Helpers.ConvertShortHashToHex(a);
+
+            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
+            // userId = "user-" + Helpers.HashToHex(dto.Email.ToLower());
+
+            FileRecord fr = await CDPDB.GetFile(dto.AppKey, dto.FileId, userId);
+            if (fr == null)
+            {
+                string AuditMessage = string.Format($"File not found for user {dto.Email} having fileId {dto.FileId}");
+                string AuditAction = "Decrypt failed";
+                await AuditFunctions.AddAuditsEvent(dto.AppKey, dto.FileId, dto.FileName, userId, "", AuditAction, AuditMessage);
+                return new BadRequestObjectResult(new { error = true, message = "File not found for user " + dto.Email });
+            }
+
+            if (fr.Policy.CheckAccess("None"))
+            {
+                string AuditMessage = string.Format($"{dto.Email} don't have the rights to decrypt {fr.FileName} file having {dto.FileId} id");
+                string AuditAction = "Decrypt failed";
+                await AuditFunctions.AddAuditsEvent(dto.AppKey, dto.FileId, fr.FileName, userId, "", AuditAction, AuditMessage);
+                return new BadRequestObjectResult(new { error = true, message = "Access is denied for user " + dto.Email });
+            }
+
+            KeyVaultService kvs = new KeyVaultService(Constants.KeyVaultURI);
+            fr.Policy.Key = await kvs.GetSecretAsync(fr.FileId);
+
+            string message = string.Format($"{dto.Email} decrypted {fr.FileName} file having {dto.FileId} id");
+            string action = "Decrypted";
+            await AuditFunctions.AddAuditsEvent(dto.AppKey, dto.FileId, dto.FileName, userId, "", action, message);
+
+            return new OkObjectResult(fr);
+        }
+
+        [Function("GetPoliciesForFile")]
+        public async Task<IActionResult> GetPoliciesForFile([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("GetFile invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            GetPoliciesForFileDto dto = JsonConvert.DeserializeObject<GetPoliciesForFileDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            List<FileRecord> fr = await CDPDB.GetPoliciesForFile(dto.AppKey, dto.FileId);
+            if (fr == null)
+            {
+                return new BadRequestObjectResult(new { error = true, message = "File not found " + dto.FileId });
+            }
+            KeyVaultService kvs = new KeyVaultService(Constants.KeyVaultURI);
+            string aesKey = await kvs.GetSecretAsync(dto.FileId);
+            foreach (var f in fr)
+            {
+                f.Policy.Key = aesKey;
+            }
+
+            return new OkObjectResult(fr);
+        }
+
+        [Function("DeleteRegisteredUserPolicies")]
+        public async Task<IActionResult> DeleteRegisteredUserPolicies([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Deleting Registered User invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            DeleteRegisteredUserDto deleteContactdto = JsonConvert.DeserializeObject<DeleteRegisteredUserDto>(requestBody);
+            if (deleteContactdto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(deleteContactdto.UserEmail.ToLower());
+            await CDPDB.revokeRegisteredUserPolicies(deleteContactdto.AppKey, deleteContactdto.UserEmail.ToLower(), deleteContactdto.ContactEmail.ToLower(), deleteContactdto.AdminEmail.ToLower());
+
+            return new OkObjectResult(true);
+        }
+
+        #endregion
+
+        #region Background Processing Support
+
+        public static async Task<Job> AddAuditsBatchedEvent(List<AuditEventMetadata> auditEvents, string appKey)
+        {
+            using (var mt = new MethodTimer("AddAuditsBatchedEvent"))
+            {
+                string jobMeta = JsonConvert.SerializeObject(auditEvents);
+                string jobId = Guid.NewGuid().ToString();
+                Job job = new Job { AppKey = appKey, EventType = JobType.AddAuditsBatch, Id = jobId, JobMetadata = jobMeta };
+                return job;
+            }
+        }
+
+        public static async Task<Job> AddKeyVaultBatchedEvent(List<KeyVaultEvent> vaultEvents, string appKey)
+        {
+            using (var mt = new MethodTimer("AddKeyVaultBatchedEvent"))
+            {
+                string jobMeta = JsonConvert.SerializeObject(vaultEvents);
+                string jobId = Guid.NewGuid().ToString();
+                Job job = new Job { AppKey = appKey, EventType = JobType.KeyVaultInsertionBatch, Id = jobId, JobMetadata = jobMeta };
+                //await MetaProcessor.PublishJob(job);
+                return job;
+            }
+        }
+
+        public static async Task<string> AddKeyVaultEvent(string fileId, string aesKey, string appKey)
+        {
+            using (var mt = new MethodTimer("AddKeyVaultEvent"))
+            {
+                KeyVaultEvent vaultEvent = new KeyVaultEvent { AESKey = aesKey, FileId = fileId };
+                string jobMeta = JsonConvert.SerializeObject(vaultEvent);
+                string jobId = Guid.NewGuid().ToString();
+                Job keyVaultJob = new Job { AppKey = appKey, EventType = JobType.KeyVaultInsertion, Id = jobId, JobMetadata = jobMeta };
+                await MetaProcessor.PublishJob(keyVaultJob);
+                return jobId;
+            }
+        }
+
+        #endregion
+    }
+}

CDP-LTS-80/FileRevision.cs → CDP-LTS-80/CDPCore/FileRevision.cs

@@ -2,8 +2,11 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Azure.Cosmos;
 using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 using System.Net;
+using FromBodyAttribute = Microsoft.Azure.Functions.Worker.Http.FromBodyAttribute;
 
 namespace CDP
 {
@@ -29,6 +32,15 @@ namespace CDP
             return new CosmosClient(uri, authKey);
         }
 
+        public static async Task<RevisionEntry> GetFileRevisionInternal(string appKey, string fileId)
+        {
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+            string id = Helpers.HashAndShortenText(appKey + fileId);
+            PartitionKey key = new PartitionKeyBuilder().Add(appKey).Add(fileId).Build();
+            var item = await container.ReadItemAsync<RevisionEntry>(id, key);
+            return item.Resource;
+        }
+
         static internal async Task<IActionResult> AddRevisionInternal(AddRevisionDocumentDto doc, ILogger _logger)
         {
             try
@@ -60,6 +72,21 @@ namespace CDP
             }
         }
 
+        public async static Task<string> AddMailBodyRevision(RevisionDocument document, CompoundDocument compoundDocument, string AppKey, string FileId)
+        {
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+            List<RevisionDocument> documents = new List<RevisionDocument> { document };
+            RevisionEntry entry = new RevisionEntry
+            {
+                AppKey = AppKey,
+                FileId = FileId,
+                FileRevisions = documents,
+                CompoundDocument = compoundDocument
+            };
+            ItemResponse<RevisionEntry> item = await container.CreateItemAsync<RevisionEntry>(entry);
+            return item.Resource.id;
+        }
+
         private async static Task<RevisionEntry> AddOrUpdateRevision(Container container, string id, PartitionKey partitionKey, RevisionDocument document, string appKey, string fileId)
         {
             try
@@ -103,6 +130,54 @@ namespace CDP
 
         }
 
+        #region Versioning Functions
+
+        [Function("AddRevisionDocument")]
+        public async Task<IActionResult> AddRevisionDocument([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("C# HTTP trigger function processed a request for update revision");
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            AddRevisionDocumentDto doc;
+            try
+            {
+                doc = JsonConvert.DeserializeObject<AddRevisionDocumentDto>(requestBody);
+                _logger.LogInformation(doc.RevisionId + doc.AppKey + doc.FileId);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex.ToString());
+                return new BadRequestObjectResult("Invalid request format");
+            }
+
+            return await AddRevisionInternal(doc, _logger);
+        }
+
+        [Function("GetFileRevisions")]
+        public async Task<IActionResult> GetFileRevisions([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, [FromBody] GetRevisionsDto reqDto)
+        {
+            _logger.LogInformation("C# HTTP trigger function processed a request for update revision", JsonConvert.SerializeObject(reqDto));
+
+            try
+            {
+
+                var item = await GetFileRevisionInternal(reqDto.AppKey, reqDto.FileId);
+                if (item.CompoundDocument != null)
+                {
+                    return new OkObjectResult(item);
+                }
+                List<RevisionDocument> revisions = item.FileRevisions;
+                //_logger.LogInformation("docs: {0}", JsonConvert.SerializeObject(revisions));
+
+                return new OkObjectResult(revisions);
+            }
+            catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                return new StatusCodeResult(404);
+            }
+        }
+
+        #endregion
+
     }
 
     public class GetRevisionsDto
@@ -128,6 +203,7 @@ namespace CDP
         public required string Comments { get; set; }
     }
 
+
     public class RevisionEntry
     {
         public string id
@@ -141,5 +217,13 @@ namespace CDP
         public required string AppKey { get; set; }
         public required string FileId { get; set; }
         public List<RevisionDocument> FileRevisions { get; set; }
+        public CompoundDocument? CompoundDocument { get; set; }
+    }
+    public class CompoundDocument
+    {
+        public required string Id { get; set; }
+        public required string AppKey { get; set; }
+        public required string Contents { get; set; } //can be MailRecord below
+        public required string DocumentType { get; set; }
     }
 }

CDP-LTS-80/CDPCore/SignalRFunctions.cs

@@ -0,0 +1,111 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using System.Net;
+
+namespace CDP.CDPCore
+{
+    /*public class SignalRFunctions
+    {
+        private readonly ILogger<SignalRFunctions> _logger;
+
+        private static readonly HttpClient HttpClient = new();
+        private static string Etag = string.Empty;
+        private static int StarCount = 0;
+
+        public SignalRFunctions(ILogger<SignalRFunctions> log)
+        {
+            _logger = log;
+        }
+
+        [Function("index")]
+        public static HttpResponseData GetHomePage([HttpTrigger(AuthorizationLevel.Anonymous)] HttpRequestData req)
+        {
+            var response = req.CreateResponse(HttpStatusCode.OK);
+            response.WriteString(File.ReadAllText("content/index.html"));
+            response.Headers.Add("Content-Type", "text/html");
+            return response;
+        }
+
+        [Function("negotiate")]
+        public static HttpResponseData Negotiate([HttpTrigger(AuthorizationLevel.Anonymous)] HttpRequestData req,
+            [SignalRConnectionInfoInput(HubName = "cdp")] string connectionInfo)
+        {
+            var response = req.CreateResponse(HttpStatusCode.OK);
+            response.Headers.Add("Content-Type", "application/json");
+            response.WriteString(connectionInfo);
+            return response;
+        }
+
+        [Function(nameof(SendMessage))]
+        [SignalROutput(HubName = "cdp")]
+        public static async Task<SignalRMessageAction> SendMessage([HttpTrigger(AuthorizationLevel.Anonymous, "post")] HttpRequestData req)
+        {
+            string requestBody = new StreamReader(req.Body).ReadToEnd();
+            SendMessagePayload pl = JsonConvert.DeserializeObject<SendMessagePayload>(requestBody);
+            if (pl == null)
+                return null;
+
+            return await SendMessageInternal(pl);
+
+        }
+
+        public static async Task<SignalRMessageAction> SendMessageInternal(SendMessagePayload pl)
+        {
+            if (pl == null)
+                return null;
+
+            if (string.IsNullOrEmpty(pl.ConnectionId) && string.IsNullOrEmpty(pl.UserId))
+                return null;
+
+            try
+            {
+                Console.WriteLine($"Sending message to {pl.ConnectionId}");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.Message);
+            }
+
+            return new SignalRMessageAction(pl.Target)
+            {
+                Arguments = new[] { pl.Message },
+                ConnectionId = pl.ConnectionId.Trim()
+            };
+        }
+
+        //[Function("broadcast")]
+        //[SignalROutput(HubName = "cdp")]
+        //public static async Task<SignalRMessageAction> Broadcast([TimerTrigger("5 * * * * *] TimerInfo timerInfo)
+        //{
+        //    var request = new HttpRequestMessage(HttpMethod.Get, "https://api.github.com/repos/azure/azure-signalr");
+        //    request.Headers.UserAgent.ParseAdd("Serverless");
+        //    request.Headers.Add("If-None-Match", Etag);
+        //    var response = await HttpClient.SendAsync(request);
+        //    if (response.Headers.Contains("Etag"))
+        //    {
+        //        Etag = response.Headers.GetValues("Etag").First();
+        //    }
+        //    if (response.StatusCode == HttpStatusCode.OK)
+        //    {
+        //        var result = await response.Content.ReadFromJsonAsync<GitResult>();
+        //        if (result != null)
+        //        {
+        //            StarCount = result.StarCount;
+        //        }
+        //    }
+        //    return new SignalRMessageAction("newMessage", new object[] { $"Current star count of https://github.com/Azure/azure-signalr is: {StarCount}" });
+        //}
+    }*/
+
+    public class SendMessagePayload
+    {
+        public string ConnectionId { get; set; }
+        public string UserId { get; set; }
+        public string Message { get; set; }
+        public string Target { get; set; }  // this will default to 'newMessage'
+    }
+}

CDP-LTS-80/CDPLite.cs

@@ -1,207 +0,0 @@
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Azure.Cosmos;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace CDP
-{
-    public class CDPLite
-    {
-        private readonly ILogger<CDPLite> _logger;
-        private static string FileAuditContainer = "FileAudits";
-        private static string UserAuditContainer = "UserAudits";
-        private static string GroupAuditContainer = "GroupAudits";
-        private static string TenantAuditContainer = "TenantAudits";
-
-        public CDPLite(ILogger<CDPLite> log)
-        {
-            _logger = log;
-        }
-
-        internal static async Task<IActionResult> AddFileUserInternal(AddFileUserDto dto)
-        {
-            // check to see if the email has the power to add a user
-            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
-
-            FileRecord fr = await CDPDB.GetFile(dto.AppKey, dto.FileId, userId);
-            if (fr == null)
-            {
-                string message = string.Format($"{dto.Email} attempted to add/change access policy for {dto.EmailToAdd} on {dto.FileName} file having {dto.FileId} id, but didn't have ANY access");
-                Console.WriteLine(message);
-                string action = "Policy change failed";
-                await AddAudits(dto.AppKey, dto.FileId, dto.FileName, userId, "", action, message);
-
-                return new BadRequestObjectResult(new { error = true, message = "File not found for user " + dto.Email });
-            }
-
-            if ((!fr.Policy.CheckAccess("Manage")) && (!fr.Policy.CheckAccess("Owner")))
-            {
-                string message = string.Format($"{dto.Email} attempted to add/change access policy for {dto.EmailToAdd} on {dto.FileName} file having {dto.FileId} id, but didn't have manage access");
-                Console.WriteLine(message);
-                string action = "Policy change failed";
-                await AddAudits(dto.AppKey, dto.FileId, dto.FileName, userId, "", action, message);
-                return new BadRequestObjectResult(new { error = true, message = $"{dto.Email} doesn't have the rights to add a user." });
-            }
-
-            string fileId = dto.FileId;
-            string fileName = dto.FileName;
-            string userIdToAdd = "";
-
-            if (dto.EmailToAdd != "")
-            {
-                userIdToAdd = Helpers.HashAndShortenText(dto.EmailToAdd.ToLower());
-            }
-            else if (dto.Group != null)
-            {
-                userIdToAdd = dto.GroupId;
-            }
-            else if (dto.Group != null)
-            {
-                userIdToAdd = dto.GroupId;
-            }
-
-            AccessPolicy ac = new AccessPolicy()
-            {
-                Access = dto.Policy,
-                Email = dto.EmailToAdd.ToLower(),
-                Group = dto.Group,
-                GroupId = dto.GroupId,
-                Key = ""
-            };
-
-            fr = await CDPDB.UpsertFile(dto.AppKey, fileId, fileName, userIdToAdd, "", ac);
-
-            if (dto.EmailToAdd != "")
-            {
-                string message = string.Format($"{dto.Email} added/changed the access policy for User : {dto.EmailToAdd} to {dto.Policy} on {fileName} file having {fileId} id");
-                string action = "Policy change";
-                await AddAudits(dto.AppKey, fileId, fileName, userId, "", action, message);
-            }
-
-            if (dto.Group != null)
-            {
-                string message = string.Format($"{dto.Email} added/changed the access policy for Group : {dto.Group} to {dto.Policy} on {fileName} file having {fileId} id");
-                string action = "Policy change";
-                await AddAudits(dto.AppKey, fileId, fileName, "", dto.Group.id, action, message);
-            }
-            return new OkObjectResult(fr);
-        }
-
-        public static async Task AddAudits(string appKey, string fileId, string fileName, string userId, string groupid, string action, string message)
-        {
-            if (string.IsNullOrEmpty(appKey) || string.IsNullOrEmpty(fileId) || string.IsNullOrEmpty(action) || string.IsNullOrEmpty(message))
-            {
-                Console.WriteLine(string.Format("something weird? appKey, fileId, action, message: {0} {1} {2} {3}", appKey, fileId, action, message));
-                return;
-            }
-                
-
-            AuditRecord faRec = new FileAuditRecord()
-            {
-                AppKey = appKey,
-                FileId = fileId,
-                FileName = fileName,
-                UserId = userId,
-                GroupId = groupid,
-                Action = action,
-                Message = message,
-                EventTime = DateTime.UtcNow,
-            };
-            Console.WriteLine("Adding File Audit Record");
-            await AuditDB.AppendRecord(faRec.id, faRec, FileAuditContainer);
-
-            AuditRecord faRecTenant = new TenantAuditRecord()
-            {
-                AppKey = appKey,
-                FileId = fileId,
-                FileName = fileName,
-                UserId = userId,
-                GroupId = groupid,
-                Action = action,
-                Message = message,
-                EventTime = DateTime.UtcNow,
-            };
-
-            await AuditDB.AppendRecord(faRecTenant.id, faRecTenant, TenantAuditContainer);
-
-            if (!string.IsNullOrEmpty(groupid))
-            {
-                AuditRecord faRecGroup = new GroupAuditRecord()
-                {
-                    AppKey = appKey,
-                    FileId = fileId,
-                    FileName = fileName,
-                    UserId = userId,
-                    GroupId = groupid,
-                    Action = action,
-                    Message = message,
-                    EventTime = DateTime.UtcNow,
-                };
-
-                await AuditDB.AppendRecord(faRecGroup.id, faRecGroup, GroupAuditContainer);
-            }
-
-            AuditRecord faRecUser = new UserAuditRecord()
-            {
-                AppKey = appKey,
-                FileId = fileId,
-                FileName = fileName,
-                UserId = userId,
-                GroupId = groupid,
-                Action = action,
-                Message = message,
-                EventTime = DateTime.UtcNow,
-            };
-
-            await AuditDB.AppendRecord(faRecUser.id, faRecUser, UserAuditContainer);
-
-        }
-        /// <summary>
-        ///  Adds the audit record on a background thread.
-        /// </summary>
-        private static async Task AddFileAudit(AuditRecord far)
-        {
-            await AuditDB.AppendRecord(far.id, far, FileAuditContainer);
-
-        }
-
-        private static async Task AddUserAudit(AuditRecord far)
-        {
-            await AuditDB.AppendRecord(far.id, far, UserAuditContainer);
-        }
-
-        private static async Task AddTenantAudit(AuditRecord far)
-        {
-            await Task.Run(async () =>
-            {
-                try
-                {
-                    await AuditDB.AppendRecord(far.id, far, TenantAuditContainer);
-                }
-                catch (Exception e)
-                {
-                }
-            });
-
-        }
-
-        private static async Task AddGroupAudit(AuditRecord far)
-        {
-            await Task.Run(async () =>
-            {
-                try
-                {
-                    await AuditDB.AppendRecord(far.id, far, GroupAuditContainer);
-                }
-                catch (Exception e)
-                {
-                }
-            });
-
-        }
-    }
-}

CDP-LTS-80/Contacts/ContactFunctions.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Threading.Tasks;
+using Azure.Messaging.ServiceBus;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace CDP
+{
+    public class ContactFunctions
+    {
+        private readonly ILogger<ContactFunctions> _logger;
+
+        public ContactFunctions(ILogger<ContactFunctions> logger)
+        {
+            _logger = logger;
+        }
+
+        [Function("GetContacts")]
+        public async Task<IActionResult> GetContacts([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Get Contacts invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            GetContactsDto dto = JsonConvert.DeserializeObject<GetContactsDto>(requestBody);
+            if (dto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(dto.Email.ToLower());
+            // userId = "user-" + Helpers.HashToHex(dto.Email.ToLower());
+
+            List<ContactRecord> fr = await ContactsDB.GetUserContacts(dto.AppKey, userId);
+            if (fr == null)
+            {
+                return new BadRequestObjectResult(new { error = true, message = "File not found " + dto.Email });
+            }
+
+            return new OkObjectResult(fr);
+        }
+
+        [Function("AddContact")]
+        public async Task<IActionResult> AddContacts([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Add Contacts invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            AddUserContactsDto addContactdto = JsonConvert.DeserializeObject<AddUserContactsDto>(requestBody);
+            if (addContactdto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(addContactdto.EmailId.ToLower());
+            ContactRecord cr = new ContactRecord(addContactdto.ContactName, addContactdto.ContactEmail, addContactdto.ContactPhone, addContactdto.ContactAddress, addContactdto.ContactCompany);
+
+            Boolean resp = await ContactsDB.AppendRecord(addContactdto.AppKey, userId, cr);
+
+            return new OkObjectResult(resp);
+        }
+
+        [Function("DeleteContact")]
+        public async Task<IActionResult> DeleteContact([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Deleting Contact invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            DeleteUserContactsDto deleteContactdto = JsonConvert.DeserializeObject<DeleteUserContactsDto>(requestBody);
+            if (deleteContactdto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(deleteContactdto.UserEmail.ToLower());
+            Boolean resp = await ContactsDB.RemoveRecord(deleteContactdto.AppKey, userId, deleteContactdto.ContactEmail.ToLower());
+            await CDPDB.revokePolicies(deleteContactdto.AppKey, deleteContactdto.UserEmail.ToLower(), deleteContactdto.ContactEmail.ToLower());
+
+            return new OkObjectResult(resp);
+        }
+    }
+}

CDP-LTS-80/Contacts/ContactsDB.cs

@@ -0,0 +1,327 @@
+using Microsoft.Azure.Cosmos;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CDP
+{
+    public class ContactsDB
+    {
+        private static Lazy<CosmosClient> lazyClient = new Lazy<CosmosClient>(InitializeCosmosClient);
+        private static CosmosClient cosmosClient => lazyClient.Value;
+        private static string DatabaseName = "CDP";
+        private static string ContainerName = "Contacts";
+
+        private static CosmosClient InitializeCosmosClient()
+        {
+            // Perform any initialization here
+            var uri = "https://cdplite.documents.azure.com:443/";
+            var authKey = "VPbg8RpzyI3XwhC2o0dIUtYFs33ghxORCqZeNAyg8vg4HWUBjM41BUxP0qLFXEvFh6ewQY1uKv52ACDbsEN1AQ==";
+
+            return new CosmosClient(uri, authKey);
+        }
+
+        public static async Task<List<ContactRecord>> GetUserContacts(string AppKey, string UserId)
+        {
+            var results = new List<ContactRecord>();
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            // Fetch the metadata document for the customer ID
+            var metadataDocumentId = GetMetaDocumentKey(AppKey, UserId);
+            MetadataDocumentContact metadataDocument = null;
+            try
+            {
+                var metadataDocumentResponse =
+                await container.ReadItemAsync<MetadataDocumentContact>(metadataDocumentId, new PartitionKey(metadataDocumentId));
+                metadataDocument = metadataDocumentResponse.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+            }
+
+            if (metadataDocument == null)
+                return results;
+
+            // Determine the partition keys within the date range
+            var partitionKeysInDocument = metadataDocument.PartitionKeys;
+            // Fetch the audit records for each partition key within the date range
+            foreach (var partitionKey in partitionKeysInDocument)
+            {
+
+                ItemResponse<ContactsDocument> response = await container.ReadItemAsync<ContactsDocument>(partitionKey, new PartitionKey(partitionKey));
+                if (response == null)
+                    continue;
+
+                ContactsDocument t = response.Resource;
+                results.AddRange(t.Records);
+            }
+
+            return results;
+        }
+
+        public static async Task<bool> AppendRecord(string appKey, string userId, ContactRecord rec)
+        {
+            try
+            {
+                var metadataDocument = await GetMetadataDocument(appKey, userId);
+                if (metadataDocument == null)
+                    return false;
+
+                string dayKey = metadataDocument.GetLatestKey(appKey, userId);
+                ContactsDocument al = await GetContactDocument(dayKey);
+                if (al == null)
+                {
+                    al = new ContactsDocument();
+                    al.AppKey = appKey;
+                    al.UserId = userId;
+                }
+                al.Records.Add(rec);
+                await UpdateContactsDocument(al);
+                return true;
+            }
+            catch (Exception e)
+            {
+                return false;
+            }
+        }
+
+        public static async Task<bool> RemoveRecord(string appKey, string userId, string EmailId)
+        {
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            // Fetch the metadata document for the customer ID
+            var metadataDocumentId = GetMetaDocumentKey(appKey, userId);
+            MetadataDocumentContact metadataDocument = null;
+            try
+            {
+                var metadataDocumentResponse = await container.ReadItemAsync<MetadataDocumentContact>(metadataDocumentId, new PartitionKey(metadataDocumentId));
+                metadataDocument = metadataDocumentResponse.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+            }
+
+            if (metadataDocument == null)
+                return false;
+
+            // Determine the partition keys within the date range
+            var partitionKeysInDocument = metadataDocument.PartitionKeys;
+            // Fetch the audit records for each partition key within the date range
+            foreach (var partitionKey in partitionKeysInDocument)
+            {
+
+                ItemResponse<ContactsDocument> response = await container.ReadItemAsync<ContactsDocument>(partitionKey, new PartitionKey(partitionKey));
+                if (response == null)
+                    continue;
+
+                ContactsDocument t = response.Resource;
+                var originalCount = t.Records.Count;
+                t.Records = t.Records.Where(item => item.Email != EmailId).ToList();
+                var afterFilterCount = t.Records.Count;
+
+                if (originalCount != afterFilterCount)
+                {
+                    ItemResponse<ContactsDocument> updateResponse = await container.ReplaceItemAsync(
+                        item: t,
+                        id: t.id,
+                        partitionKey: new PartitionKey(partitionKey));
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        public static async Task UpdateContactsDocument(ContactsDocument al)
+        {
+            if (al.Records.Count == 0)
+                return;
+
+            List<ContactsDocument> lal = await SplitAuditlog(al);
+
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            foreach (ContactsDocument ial in lal)
+            {
+                ItemResponse<ContactsDocument> r = await container.UpsertItemAsync(ial, new PartitionKey(ial.id));
+            }
+
+            await UpdateMetadata(container, lal);
+        }
+
+        static async Task UpdateMetadata(Container container, List<ContactsDocument> lal)
+        {
+            bool update = false;
+            string pKey = GetMetaDocumentKey(lal[0].AppKey, lal[0].UserId);
+            MetadataDocument md = null;
+            try
+            {
+                ItemResponse<MetadataDocument> response =
+                    await container.ReadItemAsync<MetadataDocument>(pKey, new PartitionKey(pKey));
+                md = response.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+                md = new MetadataDocument()
+                {
+                    id = GetMetaDocumentKey(lal[0].AppKey, lal[0].UserId),
+                    PartitionKeys = new List<string>()
+                };
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+                return;
+            }
+
+            if (md == null)
+            {
+                //                Helpers.LogIt("Something ugly happened!");
+                return;
+            }
+
+            foreach (ContactsDocument log in lal)
+            {
+                if (md.PartitionKeys.Contains(log.id))
+                    continue;
+                md.PartitionKeys.Add(log.id);
+                update = true;
+            }
+
+            if (update)
+            {
+                try
+                {
+                    ItemResponse<MetadataDocument> r = await container.UpsertItemAsync(md, new PartitionKey(pKey));
+                }
+                catch (Exception e)
+                {
+                    //                    Helpers.LogIt(e.Message);   
+                    return;
+                }
+            }
+        }
+
+        static async Task<List<ContactsDocument>> SplitAuditlog(ContactsDocument al)
+        {
+            List<ContactsDocument> lal = new List<ContactsDocument>();
+            var sortedRecords = al.Records.OrderBy(record => record.EventTime).ToList();
+
+            var currentGroup = new List<ContactRecord>();
+            var currentGroupSize = 0;
+
+            int MaxDocumentSizeInBytes = 2 * 1024 * 1024; // 2MB
+
+            int index = al.Index; // start for index passed in
+            foreach (var record in sortedRecords)
+            {
+                var recordSize = record.CalculateRecordSize();
+
+                if (currentGroupSize + recordSize > MaxDocumentSizeInBytes)
+                {
+                    ContactsDocument i = new ContactsDocument();
+                    i.Index = index++;
+                    i.Records = currentGroup;
+                    i.AppKey = al.AppKey;
+                    i.UserId = al.UserId;
+                    lal.Add(i);
+
+                    currentGroup = new List<ContactRecord>();
+                    currentGroupSize = 0;
+                }
+
+                currentGroup.Add(record);
+                currentGroupSize += recordSize;
+            }
+
+            if (currentGroup.Any())
+            {
+                ContactsDocument i = new ContactsDocument();
+                i.Index = index++;
+                i.Records = currentGroup;
+                i.AppKey = al.AppKey;
+                i.UserId = al.UserId;
+                lal.Add(i);
+            }
+
+            return lal;
+        }
+
+
+        public static async Task<ContactsDocument> GetContactDocument(string key)
+        {
+            try
+            {
+                Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+                ItemResponse<ContactsDocument> response = await container.ReadItemAsync<ContactsDocument>(key, new PartitionKey(key));
+                if (response == null)
+                    return null;
+
+                ContactsDocument t = response.Resource;
+                return t;
+            }
+            catch (Exception e)
+            {
+                return null;
+            }
+        }
+
+        static async Task<MetadataDocumentContact> GetMetadataDocument(string appKey, string userId)
+        {
+            MetadataDocumentContact md = null;
+            string pKey = GetMetaDocumentKey(appKey, userId);
+            string id = GetDocumentId(appKey, userId);
+
+            PartitionKey partitionKey = new PartitionKeyBuilder()
+                    .Add(pKey)
+                    .Build();
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+            try
+            {
+                ItemResponse<MetadataDocumentContact> response =
+                    await container.ReadItemAsync<MetadataDocumentContact>(pKey, partitionKey);
+                md = response.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+                md = new MetadataDocumentContact()
+                {
+                    id = id,
+                    PartitionKeys = new List<string>()
+                };
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+                return null;
+            }
+
+            if (md == null)
+            {
+                //                Helpers.LogIt("Something ugly happened!");
+                return null;
+            }
+            return md;
+        }
+
+        static string GetMetaDocumentKey(string appKey, string userId)
+        {
+            return $"{appKey}-{userId}-meta";
+        }
+
+        static string GetDocumentId(string appKey, string userId)
+        {
+            return $"{appKey}-{userId}";
+        }
+    }
+}

CDP-LTS-80/EventProcessor.cs → CDP-LTS-80/EventBased/EventProcessor.cs

@@ -77,7 +77,7 @@ namespace CDP
         internal async Task ProcessAuditEvent(Job job)
         {
             AuditEventMetadata auditEventMetadata = JsonSerializer.Deserialize<AuditEventMetadata>(job.JobMetadata);
-            await CDPLite.AddAudits(job.AppKey, auditEventMetadata.FileId, auditEventMetadata.FileName, auditEventMetadata.UserId, "", auditEventMetadata.Action, auditEventMetadata.Message);
+            await AuditFunctions.AddAudits(job.AppKey, auditEventMetadata.FileId, auditEventMetadata.FileName, auditEventMetadata.UserId, "", auditEventMetadata.Action, auditEventMetadata.Message);
             return;
         }
 
@@ -202,8 +202,9 @@ namespace CDP
     public class MailRecord
     {
         public required string AppKey { get; set; }
-        public required string MailId {  get; set; }
         public required string SenderEmail { get; set; }
+        public required string MailId { get; set; } // this is actually body Id
+        public required string BodyContent { get; set; } // this is body content
         public required List<string> Attachments { get; set; }
         public required List<string> ReceiverEmails { get; set; }
         public required List<AttachmentDetails> AttachmentDetails { get; set; }

CDP-LTS-80/EventBased/MetaProcessor.cs

@@ -0,0 +1,311 @@
+using Azure.Messaging.ServiceBus;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Text.Json;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Azure.Storage.Blobs;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Azure.Functions.Worker;
+using System.Net;
+using Microsoft.DurableTask;
+using Microsoft.DurableTask.Client;
+using FromBodyAttribute = Microsoft.Azure.Functions.Worker.Http.FromBodyAttribute;
+
+namespace CDP
+{
+    public class MetaProcessor
+    {
+        private readonly ILogger<MetaProcessor> _logger;
+        private static Lazy<ServiceBusClient> lazyBusClient = new Lazy<ServiceBusClient>(InitializeServiceBusClient);
+        private static ServiceBusClient _serviceBusClient = lazyBusClient.Value;
+
+        public MetaProcessor(ILogger<MetaProcessor> logger)
+        {
+            _logger = logger;
+        }
+
+        private static ServiceBusClient InitializeServiceBusClient()
+        {
+            return new ServiceBusClient(Constants.SvcBusConnectionString);
+        }
+
+        public static async Task PublishBatchJobs(List<Job> jobs)
+        {
+            //Job[] jobsCopy = jobs.ToArray();
+            var sender = _serviceBusClient.CreateSender("mail-events-queue");
+            ServiceBusMessageBatch batch = await sender.CreateMessageBatchAsync();
+            jobs.ForEach(job =>
+            {
+                var msg = new ServiceBusMessage(JsonSerializer.Serialize(job));
+                bool isAdded = batch.TryAddMessage(msg);
+            });
+            await sender.SendMessagesAsync(batch);
+
+        }
+
+        public static async Task PublishJob(Job job)
+        {
+            var sender = _serviceBusClient.CreateSender("mail-events-queue");
+            ServiceBusMessage msg = new ServiceBusMessage(JsonSerializer.Serialize(job));
+            await sender.SendMessageAsync(msg);
+        }
+        [Function("MailMetaProcessor")]
+        public async Task<Job> MailMetaProcessor([ActivityTrigger] MailRecord mailRecord, FunctionContext functionContext)
+        {
+            Job job = await TriggerMailProcessor(mailRecord);
+            return job;
+        }
+
+        [Function("MailMetaOrchestrator")]
+        public async Task<Job> MailMetaOrchestrator([OrchestrationTrigger] TaskOrchestrationContext ctx)
+        {
+            try
+            {
+                MailRecord record = ctx.GetInput<MailRecord>();
+                string jobId = await ctx.CallActivityAsync<string>(nameof(AddProtectionAudits), record);
+                Job job = await ctx.CallActivityAsync<Job>(nameof(MailMetaProcessor), record);
+                string revId = await ctx.CallActivityAsync<string>(nameof(AddMailCompundRevision), record);
+                return job;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex.Message);
+                return new Job() { AppKey = "", EventType = JobType.MailMetaProcessing, Id = "failed" };
+            }
+        }
+
+        [Function("AddMailMetaDurable")]
+        public async Task<HttpResponseData> AddMailMetaDurable([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req,
+            [FromBody] MailRecord recordMail,
+            [DurableClient] DurableTaskClient client)
+        {
+            bool isValid = true;
+            //bool isValid = await CDPLite.CheckJwtRequest(req);
+            
+            if (!isValid)
+            {
+                HttpResponseData res = req.CreateResponse(HttpStatusCode.Unauthorized);
+                return res;
+            }
+            try
+            {
+                string instanceId = await client.ScheduleNewOrchestrationInstanceAsync(nameof(MailMetaOrchestrator), recordMail);
+                _logger.LogInformation("created instance: {0}", instanceId);
+                return client.CreateCheckStatusResponse(req, instanceId);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex.ToString());
+                HttpResponseData response = req.CreateResponse(HttpStatusCode.InternalServerError);
+                return response;
+            }
+        }
+
+        [Function("AddMailMeta")]
+        public async Task<ActionResult> AddMailMeta([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, [FromBody] MailRecord recordMail)
+        {
+            bool isValid = true;
+            //bool isValid = await CDPLite.CheckJwtRequest(req);
+            if (!isValid)
+            {
+                return new UnauthorizedObjectResult("Jwt has expired please validate");
+            }
+            try
+            {
+                MailRecord mailRecord = recordMail;
+                //_logger.LogInformation(JsonSerializer.Serialize(recordMail));
+                string res = await TriggerProtectionAuditsJob(mailRecord);
+                res = await AddMailCompoundRevisionInternal(mailRecord);
+                _logger.LogInformation($"protection audits? {res}");
+                Job job = await TriggerMailProcessor(mailRecord);
+                return new OkObjectResult(job);
+
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex.ToString());
+                return new StatusCodeResult(500);
+            }
+
+        }
+
+        [Function("AddMailForwardingRequest")]
+        public async Task<IActionResult> AddMailForwardingRequest([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req)
+        {
+            ForwardMailDto forwardMail;
+            try
+            {
+                string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+                _logger.LogInformation("requestbody: {0}", requestBody);
+                forwardMail = JsonSerializer.Deserialize<ForwardMailDto>(requestBody);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogInformation(ex.ToString());
+                return new StatusCodeResult(500);
+            }
+            string appKey = forwardMail.AppKey;
+            var userId = Helpers.HashAndShortenText(forwardMail.SenderEmail.ToLower());
+            var fr = await CDPDB.GetFile(appKey, forwardMail.MailId, userId);
+            if ((!fr.Policy.CheckAccess("Manage")) && (!fr.Policy.CheckAccess("Owner")))
+            {
+                return new UnauthorizedObjectResult("User doesn't have rights to forward this email");
+            }
+            RevisionEntry entry = await FileRevision.GetFileRevisionInternal(forwardMail.AppKey, forwardMail.MailId);
+            if (entry.CompoundDocument == null)
+            {
+                return new BadRequestObjectResult("IncorrectMailId");
+            }
+            MailMetaRecord mailRecord = JsonSerializer.Deserialize<MailMetaRecord>(entry.CompoundDocument.Contents);
+
+            mailRecord.AttachmentDetails.ForEach(async (att) =>
+            {
+                forwardMail.ReceiverEmails.ForEach(async (receiver) =>
+                {
+                    await AddMailForwardPolicy(appKey, att.FileId, att.FileName, forwardMail.SenderEmail, receiver, "Read");
+                });
+            });
+
+            forwardMail.ReceiverEmails.ForEach(async (receiver) =>
+            {
+                await AddMailForwardPolicy(appKey, mailRecord.BodyId, string.Concat(mailRecord.BodyId, "-", "MailBody"), forwardMail.SenderEmail, receiver, "Read");
+            });
+
+            return new OkObjectResult("Done");
+        }
+
+        internal async Task AddMailForwardPolicy(string appKey, string fileId, string fileName, string sender, string receiver, string policy = "Read")
+        {
+            AddFileUserDto addFileUserDto = new AddFileUserDto
+            {
+                AppKey = appKey,
+                FileId = fileId,
+                FileName = fileName,
+                Email = sender,
+                EmailToAdd = receiver,
+                Policy = policy
+            };
+            await CDPLite.AddFileUserInternal(addFileUserDto);
+        }
+
+        [Function("AddMailCompundRevision")]
+        public async Task<string> AddMailCompundRevision([ActivityTrigger] MailRecord record, FunctionContext context)
+        {
+
+            return await AddMailCompoundRevisionInternal(record);
+        }
+
+        internal async Task<string> AddMailCompoundRevisionInternal(MailRecord record)
+        {
+            RevisionDocument doc = new RevisionDocument
+            {
+                RevisionId = Guid.NewGuid().ToString(),
+                RevisionDate = DateTime.UtcNow,
+                EditedBy = record.SenderEmail,
+                Comments = "MailBody"
+            };
+            MailMetaRecord mailMeta = new MailMetaRecord
+            {
+                AppKey = record.AppKey,
+                SenderEmail = record.SenderEmail,
+                BodyContent = record.BodyContent,
+                BodyId = record.MailId,
+                AttachmentDetails = record.AttachmentDetails
+            };
+            CompoundDocument compound = new CompoundDocument
+            {
+                Id = Guid.NewGuid().ToString(),
+                AppKey = record.AppKey,
+                Contents = JsonSerializer.Serialize(mailMeta),
+                DocumentType = "MailRecord"
+            };
+
+            string revisionId = await FileRevision.AddMailBodyRevision(doc, compound, record.AppKey, record.MailId);
+            return revisionId;
+        }
+
+        [Function("AddProtectionAudits")]
+        public async Task<string> AddProtectionAudits([ActivityTrigger] MailRecord record, FunctionContext functionContext)
+        {
+            var res = await TriggerProtectionAuditsJob(record);
+            return res;
+        }
+
+        internal async Task<string> TriggerProtectionAuditsJob(MailRecord record)
+        {
+            try
+            {
+                string userId = Helpers.HashAndShortenText(record.SenderEmail);
+                string message = string.Format($"{record.SenderEmail} protected an email body with id: {record.MailId}.");
+                List<AuditEventMetadata> auditEvents = new List<AuditEventMetadata>();
+                AuditEventMetadata auditEvent = new AuditEventMetadata { Action = "Addded", FileId = record.MailId, FileName = "Outlook-Mail-Body", Message = message, UserId = userId };
+                auditEvents.Add(auditEvent);
+                record.AttachmentDetails.ForEach((att) =>
+                {
+                    message = string.Format($"{record.SenderEmail} protected the email attachment {att.FileName} with id: {att.FileId}.");
+                    AuditEventMetadata attachmentEvent = new AuditEventMetadata { Action = "Added", FileId = att.FileId, FileName = att.FileName, Message = message, UserId = userId };
+                    auditEvents.Add(attachmentEvent);
+                });
+                Job job = await CDPLite.AddAuditsBatchedEvent(auditEvents, record.AppKey);
+                await PublishJob(job);
+                _logger.LogInformation(JsonSerializer.Serialize(job));
+                return "Done";
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex.ToString());
+                throw ex;
+            }
+        }
+
+        internal async Task<Job> TriggerMailProcessor(MailRecord mailRecord)
+        {
+            string Connection = Constants.StorageConnString;
+            var blobSvc = new BlobServiceClient(Connection);
+            var blobClient = blobSvc.GetBlobContainerClient(mailRecord.AppKey.ToLower());
+
+            if (!blobClient.Exists())
+            {
+                await blobClient.CreateAsync();
+            }
+            byte[] fileBytes = System.Text.Encoding.UTF8.GetBytes(JsonSerializer.Serialize(mailRecord));
+            string jobId = Guid.NewGuid().ToString();
+            var blob = blobClient.GetBlobClient(jobId);
+            Stream fs = new MemoryStream(fileBytes);
+            try
+            {
+                await blob.UploadAsync(fs);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex.ToString());
+                throw ex;
+            }
+
+            Job job = new Job { AppKey = mailRecord.AppKey, EventType = JobType.MailMetaProcessing, Id = jobId };
+            await PublishJob(job);
+            return job;
+        }
+    }
+
+    public class MailMetaRecord
+    {
+        public required string AppKey { get; set; }
+        public required string SenderEmail { get; set; }
+        public required string BodyId { get; set; }
+        public required string BodyContent { get; set; } //encrypted body
+        public required List<AttachmentDetails> AttachmentDetails { get; set; }
+    }
+
+    public class ForwardMailDto
+    {
+        public string SenderEmail { get; set; }
+        public List<string> ReceiverEmails { get; set; }
+        public string MailId { get; set; }
+        public string AppKey { get; set; }
+    }
+}

CDP-LTS-80/Groups/GroupsDB.cs

@@ -0,0 +1,388 @@
+using Microsoft.Azure.Cosmos;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CDP
+{
+    public class GroupsDB
+    {
+        private static Lazy<CosmosClient> lazyClient = new Lazy<CosmosClient>(InitializeCosmosClient);
+        private static CosmosClient cosmosClient => lazyClient.Value;
+        private static string DatabaseName = "CDP";
+        private static string ContainerName = "Groups";
+
+        private static CosmosClient InitializeCosmosClient()
+        {
+            // Perform any initialization here
+            var uri = "https://cdplite.documents.azure.com:443/";
+            var authKey = "VPbg8RpzyI3XwhC2o0dIUtYFs33ghxORCqZeNAyg8vg4HWUBjM41BUxP0qLFXEvFh6ewQY1uKv52ACDbsEN1AQ==";
+
+            return new CosmosClient(uri, authKey);
+        }
+
+        public static async Task<List<GroupsRecord>> GetUserGroups(string AppKey, string UserId)
+        {
+            var results = new List<GroupsRecord>();
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            // Fetch the metadata document for the customer ID
+            var metadataDocumentId = GetMetaDocumentKey(AppKey, UserId);
+            MetadataDocumentGroups metadataDocument = null;
+            try
+            {
+                var metadataDocumentResponse =
+                await container.ReadItemAsync<MetadataDocumentGroups>(metadataDocumentId, new PartitionKey(metadataDocumentId));
+                metadataDocument = metadataDocumentResponse.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+            }
+
+            if (metadataDocument == null)
+                return results;
+
+            // Determine the partition keys within the date range
+            var partitionKeysInDocument = metadataDocument.PartitionKeys;
+            // Fetch the audit records for each partition key within the date range
+            foreach (var partitionKey in partitionKeysInDocument)
+            {
+
+                ItemResponse<GroupsDocument> response = await container.ReadItemAsync<GroupsDocument>(partitionKey, new PartitionKey(partitionKey));
+                if (response == null)
+                    continue;
+
+                GroupsDocument t = response.Resource;
+                results.AddRange(t.Records);
+            }
+
+            return results;
+        }
+
+        public static async Task<bool> RemoveGroup(string appKey, string userId, string groupId)
+        {
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            // Fetch the metadata document for the customer ID
+            var metadataDocumentId = GetMetaDocumentKey(appKey, userId);
+            MetadataDocumentContact metadataDocument = null;
+            try
+            {
+                var metadataDocumentResponse = await container.ReadItemAsync<MetadataDocumentContact>(metadataDocumentId, new PartitionKey(metadataDocumentId));
+                metadataDocument = metadataDocumentResponse.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+            }
+
+            if (metadataDocument == null)
+                return false;
+
+            // Determine the partition keys within the date range
+            var partitionKeysInDocument = metadataDocument.PartitionKeys;
+            // Fetch the audit records for each partition key within the date range
+            foreach (var partitionKey in partitionKeysInDocument)
+            {
+
+                ItemResponse<GroupsDocument> response = await container.ReadItemAsync<GroupsDocument>(partitionKey, new PartitionKey(partitionKey));
+                if (response == null)
+                    continue;
+
+                GroupsDocument t = response.Resource;
+                var originalCount = t.Records.Count;
+                t.Records = t.Records.Where(item => item.id != groupId).ToList();
+                var afterFilterCount = t.Records.Count;
+
+                if (originalCount != afterFilterCount)
+                {
+                    ItemResponse<GroupsDocument> updateResponse = await container.ReplaceItemAsync(
+                        item: t,
+                        id: t.id,
+                        partitionKey: new PartitionKey(partitionKey));
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        public static async Task<bool> AppendGroup(string appKey, string userId, String grpName, string grpDescription, List<ContactRecord> rec)
+        {
+            try
+            {
+                var metadataDocument = await GetMetadataDocument(appKey, userId);
+                if (metadataDocument == null)
+                    return false;
+
+                string dayKey = metadataDocument.GetLatestKey(appKey, userId);
+                GroupsDocument al = await GetGroupDocument(dayKey);
+                if (al == null)
+                {
+                    al = new GroupsDocument();
+                    al.AppKey = appKey;
+                    al.UserId = userId;
+                }
+                GroupsRecord gr = new GroupsRecord(grpName, grpDescription, rec);
+                al.Records.Add(gr);
+                await UpdateGroupsDocument(al);
+                return true;
+            }
+            catch (Exception e)
+            {
+                return false;
+            }
+        }
+
+        public static async Task<bool> UpdateGroup(string appKey, string userId, string grpId, string grpName, string grpDescription, List<ContactRecord> contacts)
+        {
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            // Fetch the metadata document for the customer ID
+            var metadataDocumentId = GetMetaDocumentKey(appKey, userId);
+            MetadataDocumentContact metadataDocument = null;
+            try
+            {
+                var metadataDocumentResponse = await container.ReadItemAsync<MetadataDocumentContact>(metadataDocumentId, new PartitionKey(metadataDocumentId));
+                metadataDocument = metadataDocumentResponse.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+            }
+
+            if (metadataDocument == null)
+                return false;
+
+            // Determine the partition keys within the date range
+            var partitionKeysInDocument = metadataDocument.PartitionKeys;
+            // Fetch the audit records for each partition key within the date range
+            foreach (var partitionKey in partitionKeysInDocument)
+            {
+
+                ItemResponse<GroupsDocument> response = await container.ReadItemAsync<GroupsDocument>(partitionKey, new PartitionKey(partitionKey));
+                if (response == null)
+                    continue;
+
+                GroupsDocument t = response.Resource;
+                bool groupFound = false;
+                foreach (GroupsRecord gr in t.Records)
+                {
+                    if (gr.id == grpId)
+                    {
+                        gr.Name = grpName != "" ? grpName : gr.Name;
+                        gr.Description = grpDescription != "" ? grpDescription : gr.Description;
+                        gr.Contacts = contacts;
+                        groupFound = true;
+                        break;
+                    }
+                }
+
+                if (groupFound)
+                {
+                    ItemResponse<GroupsDocument> updateResponse = await container.ReplaceItemAsync(
+                        item: t,
+                        id: t.id,
+                        partitionKey: new PartitionKey(partitionKey));
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        public static async Task UpdateGroupsDocument(GroupsDocument al)
+        {
+            if (al.Records.Count == 0)
+                return;
+
+            List<GroupsDocument> lal = await SplitUserGroups(al);
+
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+
+            foreach (GroupsDocument ial in lal)
+            {
+                ItemResponse<GroupsDocument> r = await container.UpsertItemAsync(ial, new PartitionKey(ial.id));
+            }
+
+            await UpdateMetadata(container, lal);
+        }
+
+        static async Task UpdateMetadata(Container container, List<GroupsDocument> lal)
+        {
+            bool update = false;
+            string pKey = GetMetaDocumentKey(lal[0].AppKey, lal[0].UserId);
+            MetadataDocument md = null;
+            try
+            {
+                ItemResponse<MetadataDocument> response =
+                    await container.ReadItemAsync<MetadataDocument>(pKey, new PartitionKey(pKey));
+                md = response.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+                md = new MetadataDocument()
+                {
+                    id = GetMetaDocumentKey(lal[0].AppKey, lal[0].UserId),
+                    PartitionKeys = new List<string>()
+                };
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+                return;
+            }
+
+            if (md == null)
+            {
+                //                Helpers.LogIt("Something ugly happened!");
+                return;
+            }
+
+            foreach (GroupsDocument log in lal)
+            {
+                if (md.PartitionKeys.Contains(log.id))
+                    continue;
+                md.PartitionKeys.Add(log.id);
+                update = true;
+            }
+
+            if (update)
+            {
+                try
+                {
+                    ItemResponse<MetadataDocument> r = await container.UpsertItemAsync(md, new PartitionKey(pKey));
+                }
+                catch (Exception e)
+                {
+                    //                    Helpers.LogIt(e.Message);   
+                    return;
+                }
+            }
+        }
+
+        static async Task<List<GroupsDocument>> SplitUserGroups(GroupsDocument al)
+        {
+            List<GroupsDocument> lal = new List<GroupsDocument>();
+            var sortedRecords = al.Records.OrderBy(record => record.EventTime).ToList();
+
+            var currentGroup = new List<GroupsRecord>();
+            var currentGroupSize = 0;
+
+            int MaxDocumentSizeInBytes = 2 * 1024 * 1024; // 2MB
+
+            int index = al.Index; // start for index passed in
+            foreach (var record in sortedRecords)
+            {
+                var recordSize = record.CalculateRecordSize();
+
+                if (currentGroupSize + recordSize > MaxDocumentSizeInBytes)
+                {
+                    GroupsDocument i = new GroupsDocument();
+                    i.Index = index++;
+                    i.Records = currentGroup;
+                    i.AppKey = al.AppKey;
+                    i.UserId = al.UserId;
+                    lal.Add(i);
+
+                    currentGroup = new List<GroupsRecord>();
+                    currentGroupSize = 0;
+                }
+
+                currentGroup.Add(record);
+                currentGroupSize += recordSize;
+            }
+
+            if (currentGroup.Any())
+            {
+                GroupsDocument i = new GroupsDocument();
+                i.Index = index++;
+                i.Records = currentGroup;
+                i.AppKey = al.AppKey;
+                i.UserId = al.UserId;
+                lal.Add(i);
+            }
+
+            return lal;
+        }
+
+
+        public static async Task<GroupsDocument> GetGroupDocument(string key)
+        {
+            try
+            {
+                Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+                ItemResponse<GroupsDocument> response = await container.ReadItemAsync<GroupsDocument>(key, new PartitionKey(key));
+                if (response == null)
+                    return null;
+
+                GroupsDocument t = response.Resource;
+                return t;
+            }
+            catch (Exception e)
+            {
+                return null;
+            }
+        }
+
+        static async Task<MetadataDocumentGroups> GetMetadataDocument(string appKey, string userId)
+        {
+            MetadataDocumentGroups md = null;
+            string pKey = GetMetaDocumentKey(appKey, userId);
+            string id = GetDocumentId(appKey, userId);
+
+            PartitionKey partitionKey = new PartitionKeyBuilder()
+                    .Add(pKey)
+                    .Build();
+            Container container = cosmosClient.GetContainer(DatabaseName, ContainerName);
+            try
+            {
+                ItemResponse<MetadataDocumentGroups> response =
+                    await container.ReadItemAsync<MetadataDocumentGroups>(pKey, partitionKey);
+                md = response.Resource;
+            }
+            catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+                md = new MetadataDocumentGroups()
+                {
+                    id = id,
+                    PartitionKeys = new List<string>()
+                };
+            }
+            catch (Exception e)
+            {
+                //                Helpers.LogIt(e.Message);
+                return null;
+            }
+
+            if (md == null)
+            {
+                //                Helpers.LogIt("Something ugly happened!");
+                return null;
+            }
+            return md;
+        }
+
+        static string GetMetaDocumentKey(string appKey, string userId)
+        {
+            return $"{appKey}-{userId}-meta";
+        }
+
+        static string GetDocumentId(string appKey, string userId)
+        {
+            return $"{appKey}-{userId}";
+        }
+    }
+}

CDP-LTS-80/Groups/GroupsFunctions.cs

@@ -0,0 +1,107 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace CDP
+{
+    public class GroupsFunctions
+    {
+        private readonly ILogger<GroupsFunctions> _logger;
+
+        public GroupsFunctions(ILogger<GroupsFunctions> logger)
+        {
+            _logger = logger;
+        }
+
+        [Function("AddGroup")]
+        public async Task<IActionResult> AddGroup([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Adding a new group invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            AddUserGroupDto addUserGroupDto = JsonConvert.DeserializeObject<AddUserGroupDto>(requestBody);
+            if (addUserGroupDto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(addUserGroupDto.EmailId.ToLower());
+
+            List<ContactRecord> contacts = new List<ContactRecord>();
+            foreach (Contact contact in addUserGroupDto.Contacts)
+            {
+                ContactRecord cr = new ContactRecord(contact.Name, contact.Email, contact.Phone, contact.Address, contact.Company);
+                contacts.Add(cr);
+            }
+
+            Boolean resp = await GroupsDB.AppendGroup(addUserGroupDto.AppKey, userId, addUserGroupDto.GroupName, addUserGroupDto.GroupDescription, contacts);
+
+            return new OkObjectResult(resp);
+        }
+
+        [Function("GetGroups")]
+        public async Task<IActionResult> GetGroups([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Getting groups invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            GetUserGroupDto getUserGroupDto = JsonConvert.DeserializeObject<GetUserGroupDto>(requestBody);
+            if (getUserGroupDto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(getUserGroupDto.EmailId.ToLower());
+
+            List<GroupsRecord> fr = await GroupsDB.GetUserGroups(getUserGroupDto.AppKey, userId);
+            if (fr == null)
+            {
+                return new BadRequestObjectResult(new { error = true, message = "groups not found " + getUserGroupDto.EmailId });
+            }
+
+            return new OkObjectResult(fr);
+        }
+
+        [Function("DeleteGroup")]
+        public async Task<IActionResult> DeleteGroup([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Deleting Group invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            DeleteGroupDto deleteGroupdto = JsonConvert.DeserializeObject<DeleteGroupDto>(requestBody);
+            if (deleteGroupdto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(deleteGroupdto.UserEmail.ToLower());
+            Boolean resp = await GroupsDB.RemoveGroup(deleteGroupdto.AppKey, userId, deleteGroupdto.GroupId);
+            return new OkObjectResult(resp);
+        }
+
+        [Function("UpdateGroup")]
+        public async Task<IActionResult> UpdateGroup([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+        {
+            _logger.LogInformation("Updating Group invoked");
+
+            // Convert the JSON payload to a string
+            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
+            UpdateGroupDto updateUserGroupDto = JsonConvert.DeserializeObject<UpdateGroupDto>(requestBody);
+            if (updateUserGroupDto == null)
+                return new BadRequestObjectResult(new { error = true, message = "Parse error." });
+
+            string userId = Helpers.HashAndShortenText(updateUserGroupDto.UserEmail.ToLower());
+
+            List<ContactRecord> contacts = new List<ContactRecord>();
+            foreach (Contact contact in updateUserGroupDto.Contacts)
+            {
+                ContactRecord cr = new ContactRecord(contact.Name, contact.Email, contact.Phone, contact.Address, contact.Company);
+                contacts.Add(cr);
+            }
+
+            Boolean resp = await GroupsDB.UpdateGroup(updateUserGroupDto.AppKey, userId, updateUserGroupDto.GroupId, updateUserGroupDto.GroupName, updateUserGroupDto.GroupDescription, contacts);
+
+            return new OkObjectResult(resp);
+        }
+    }
+}

CDP-LTS-80/Helpers.cs

@@ -9,9 +9,48 @@ using System.Text;
 using System.Threading.Tasks;
 using Azure.Identity;
 using Azure.Security.KeyVault.Secrets;
+using System.Diagnostics;
 
 namespace CDP
 {
+    public class MethodTimer : IDisposable
+    {
+        private readonly Stopwatch stopwatch = new Stopwatch();
+        private readonly string methodName;
+
+        public MethodTimer(string methodName)
+        {
+            this.methodName = methodName;
+            Start();
+        }
+
+        public void Dispose()
+        {
+            Stop();
+        }
+
+        public void Start()
+        {
+            stopwatch.Start();
+        }
+
+        public void Stop()
+        {
+            stopwatch.Stop();
+            LogElapsedTime();
+        }
+
+        private void LogElapsedTime()
+        {
+            Dictionary<string, object> log = new Dictionary<string, object>
+            {
+                {"MethodName", methodName},
+                {"ExecutionTime", stopwatch.ElapsedMilliseconds},
+            };
+            Console.WriteLine($"\n{JsonConvert.SerializeObject(log)}\n");
+            // You can customize the logging mechanism (e.g., use a logging library, write to a file, etc.) based on your requirements.
+        }
+    }
     public class Helpers
     {
         public static byte[] GenerateAES256Key()

CDP-LTS-80/host.json

@@ -8,5 +8,8 @@
       },
       "enableLiveMetricsFilters": true
     }
+  },
+  "tracing": {
+    "consoleLevel": "verbose"
   }
 }