* Minimal changes for blob end point to allow for extension in dfs en…

…dpoint

* Added clearDb option to configurations in and set it to true in the test cases

ChangeLog.md

@@ -8,6 +8,7 @@ Blob:
 
 - Fixed issue of: blob batch subresponse is slightly different from the on from Azure serivce, which causes exception in CPP SDK.
 - Fixed issue of: setMetadata API allows invalid metadata name with hyphen.
+- Support Same features in SQL Metadata Store as Loki Metadata Store (support Blob Copy & Page Blob)
 
 ## 2023.03 Version 3.23.0
 

README.md

@@ -473,22 +473,21 @@ Azurite will refresh customized account name and key from environment variable e
 
 By default, Azurite leverages [loki](https://github.com/techfort/LokiJS) as metadata database.
 However, as an in-memory database, loki limits Azurite's scalability and data persistency.
-Set environment variable `AZURITE_DB=dialect://[username][:password][@]host:port/database` to make Azurite blob service switch to a SQL database based metadata storage, like MySql, SqlServer.
+Set environment variable `AZURITE_DB=dialect://[username][:password][@]host:port/database` to make Azurite blob service switch to a SQL database based metadata storage, like MySql, SqlServer & PostgreSQL.
 
 For example, connect to MySql or SqlServer by set environment variables:
 
 ```bash
 set AZURITE_DB=mysql://username:password@localhost:3306/azurite_blob
 set AZURITE_DB=mssql://username:password@localhost:1024/azurite_blob
+set AZURITE_DB=postgres://username:password@localhost:5432/azurite_blob
 ```
 
 When Azurite starts with above environment variable, it connects to the configured database, and creates tables if not exist.
 This feature is in preview, when Azurite changes database table schema, you need to drop existing tables and let Azurite regenerate database tables.
 
 > Note. Need to manually create database before starting Azurite instance.
 
-> Note. Blob Copy & Page Blob are not supported by SQL based metadata implementation.
-
 > Tips. Create database instance quickly with docker, for example `docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=my-secret-pw -d mysql:latest`. Grant external access and create database `azurite_blob` using `docker exec mysql mysql -u root -pmy-secret-pw -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES; create database azurite_blob;"`. Notice that, above commands are examples, you need to carefully define the access permissions in your production environment.
 
 ## HTTPS Setup

src/blob/BlobConfiguration.ts

@@ -2,10 +2,8 @@ import ConfigurationBase from "../common/ConfigurationBase";
 import { StoreDestinationArray } from "../common/persistence/IExtentStore";
 import {
   DEFAULT_BLOB_EXTENT_LOKI_DB_PATH,
-  DEFAULT_BLOB_LISTENING_PORT,
   DEFAULT_BLOB_LOKI_DB_PATH,
   DEFAULT_BLOB_PERSISTENCE_ARRAY,
-  DEFAULT_BLOB_SERVER_HOST_NAME,
   DEFAULT_ENABLE_ACCESS_LOG,
   DEFAULT_ENABLE_DEBUG_LOG
 } from "./utils/constants";
@@ -24,8 +22,8 @@ import {
  */
 export default class BlobConfiguration extends ConfigurationBase {
   public constructor(
-    host: string = DEFAULT_BLOB_SERVER_HOST_NAME,
-    port: number = DEFAULT_BLOB_LISTENING_PORT,
+    host: string,
+    port: number,
     public readonly metadataDBPath: string = DEFAULT_BLOB_LOKI_DB_PATH,
     public readonly extentDBPath: string = DEFAULT_BLOB_EXTENT_LOKI_DB_PATH,
     public readonly persistencePathArray: StoreDestinationArray = DEFAULT_BLOB_PERSISTENCE_ARRAY,

src/blob/BlobEnvironment.ts

@@ -5,11 +5,23 @@ import { dirname } from "path";
 import IBlobEnvironment from "./IBlobEnvironment";
 import {
   DEFAULT_BLOB_LISTENING_PORT,
-  DEFAULT_BLOB_SERVER_HOST_NAME
+  DEFAULT_BLOB_SERVER_HOST_NAME,
+  DEFAULT_DATA_LAKE_LISTENING_PORT,
+  DEFAULT_DATA_LAKE_SERVER_HOST_NAME
 } from "./utils/constants";
 
 if (!(args as any).config.name) {
   args
+    .option(
+      ["", "datalakeHost"],
+      "Optional. Customize listening address for blob",
+      DEFAULT_DATA_LAKE_SERVER_HOST_NAME
+    )
+    .option(
+      ["", "datalakePort"],
+      "Optional. Customize listening port for blob",
+      DEFAULT_DATA_LAKE_LISTENING_PORT
+    )
     .option(
       ["", "blobHost"],
       "Optional. Customize listening address for blob",
@@ -56,6 +68,14 @@ if (!(args as any).config.name) {
 export default class BlobEnvironment implements IBlobEnvironment {
   private flags = args.parse(process.argv);
 
+  public datalakeHost(): string | undefined {
+    return this.flags.datalakeHost;
+  }
+
+  public datalakePort(): number | undefined {
+    return this.flags.datalakePort;
+  }
+
   public blobHost(): string | undefined {
     return this.flags.blobHost;
   }

src/blob/BlobServer.ts

@@ -18,10 +18,7 @@ import BlobRequestListenerFactory from "./BlobRequestListenerFactory";
 import BlobGCManager from "./gc/BlobGCManager";
 import IBlobMetadataStore from "./persistence/IBlobMetadataStore";
 import LokiBlobMetadataStore from "./persistence/LokiBlobMetadataStore";
-
-const BEFORE_CLOSE_MESSAGE = `Azurite Blob service is closing...`;
-const BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite Blob service is closing... Critical error happens during GC.`;
-const AFTER_CLOSE_MESSAGE = `Azurite Blob service successfully closed`;
+import { DEFAULT_BLOB_LISTENING_PORT, DEFAULT_BLOB_SERVER_HOST_NAME } from "./utils/constants";
 
 /**
  * Default implementation of Azurite Blob HTTP server.
@@ -42,16 +39,27 @@ export default class BlobServer extends ServerBase implements ICleaner {
   private readonly extentStore: IExtentStore;
   private readonly accountDataStore: IAccountDataStore;
   private readonly gcManager: IGCManager;
+  private readonly BEFORE_CLOSE_MESSAGE;
+  private readonly BEFORE_CLOSE_MESSAGE_GC_ERROR;
+  private readonly AFTER_CLOSE_MESSAGE;
 
   /**
    * Creates an instance of Server.
    *
    * @param {BlobConfiguration} configuration
    * @memberof Server
    */
-  constructor(configuration?: BlobConfiguration) {
+  constructor(
+    configuration?: BlobConfiguration, 
+    metadataStoreClass: any = LokiBlobMetadataStore,
+    requestListnerFactory: any = BlobRequestListenerFactory,
+    private readonly serviceName: string = "Blob"
+  ) {
     if (configuration === undefined) {
-      configuration = new BlobConfiguration();
+      configuration = new BlobConfiguration(
+        DEFAULT_BLOB_SERVER_HOST_NAME, 
+        DEFAULT_BLOB_LISTENING_PORT
+      )
     }
 
     const host = configuration.host;
@@ -72,7 +80,7 @@ export default class BlobServer extends ServerBase implements ICleaner {
     // We can change the persistency layer implementation by
     // creating a new XXXDataStore class implementing IBlobMetadataStore interface
     // and replace the default LokiBlobMetadataStore
-    const metadataStore: IBlobMetadataStore = new LokiBlobMetadataStore(
+    const metadataStore: IBlobMetadataStore = new metadataStoreClass(
       configuration.metadataDBPath
       // logger
     );
@@ -92,7 +100,7 @@ export default class BlobServer extends ServerBase implements ICleaner {
     // We can also change the HTTP framework here by
     // creating a new XXXListenerFactory implementing IRequestListenerFactory interface
     // and replace the default Express based request listener
-    const requestListenerFactory: IRequestListenerFactory = new BlobRequestListenerFactory(
+    const requestListenerFactory: IRequestListenerFactory = new requestListnerFactory(
       metadataStore,
       extentStore,
       accountDataStore,
@@ -114,17 +122,20 @@ export default class BlobServer extends ServerBase implements ICleaner {
       extentStore,
       () => {
         // tslint:disable-next-line:no-console
-        console.log(BEFORE_CLOSE_MESSAGE_GC_ERROR);
-        logger.info(BEFORE_CLOSE_MESSAGE_GC_ERROR);
+        console.log(this.BEFORE_CLOSE_MESSAGE_GC_ERROR);
+        logger.info(this.BEFORE_CLOSE_MESSAGE_GC_ERROR);
         this.close().then(() => {
           // tslint:disable-next-line:no-console
-          console.log(AFTER_CLOSE_MESSAGE);
-          logger.info(AFTER_CLOSE_MESSAGE);
+          console.log(this.AFTER_CLOSE_MESSAGE);
+          logger.info(this.AFTER_CLOSE_MESSAGE);
         });
       },
       logger
     );
 
+    this.BEFORE_CLOSE_MESSAGE = `Azurite ${serviceName} service is closing...`;
+    this.BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite ${serviceName} service is closing... Critical error happens during GC.`;
+    this.AFTER_CLOSE_MESSAGE = `Azurite ${serviceName} service successfully closed`;
     this.metadataStore = metadataStore;
     this.extentMetadataStore = extentMetadataStore;
     this.extentStore = extentStore;
@@ -158,11 +169,11 @@ export default class BlobServer extends ServerBase implements ICleaner {
       }
       return;
     }
-    throw Error(`Cannot clean up blob server in status ${this.getStatus()}.`);
+    throw Error(`Cannot clean up ${this.serviceName} server in status ${this.getStatus()}.`);
   }
 
   protected async beforeStart(): Promise<void> {
-    const msg = `Azurite Blob service is starting on ${this.host}:${this.port}`;
+    const msg = `Azurite ${this.serviceName} service is starting on ${this.host}:${this.port}`;
     logger.info(msg);
 
     if (this.accountDataStore !== undefined) {
@@ -187,12 +198,12 @@ export default class BlobServer extends ServerBase implements ICleaner {
   }
 
   protected async afterStart(): Promise<void> {
-    const msg = `Azurite Blob service successfully listens on ${this.getHttpServerAddress()}`;
+    const msg = `Azurite ${this.serviceName} service successfully listens on ${this.getHttpServerAddress()}`;
     logger.info(msg);
   }
 
   protected async beforeClose(): Promise<void> {
-    logger.info(BEFORE_CLOSE_MESSAGE);
+    logger.info(this.BEFORE_CLOSE_MESSAGE);
   }
 
   protected async afterClose(): Promise<void> {
@@ -216,6 +227,6 @@ export default class BlobServer extends ServerBase implements ICleaner {
       await this.accountDataStore.close();
     }
 
-    logger.info(AFTER_CLOSE_MESSAGE);
+    logger.info(this.AFTER_CLOSE_MESSAGE);
   }
 }

src/blob/BlobServerFactory.ts

@@ -7,16 +7,43 @@ import BlobServer from "./BlobServer";
 import IBlobEnvironment from "./IBlobEnvironment";
 import SqlBlobConfiguration from "./SqlBlobConfiguration";
 import SqlBlobServer from "./SqlBlobServer";
-import { DEFAULT_BLOB_PERSISTENCE_PATH } from "./utils/constants";
+import { DEFAULT_BLOB_LISTENING_PORT, DEFAULT_BLOB_PERSISTENCE_PATH, DEFAULT_BLOB_SERVER_HOST_NAME } from "./utils/constants";
 import {
   DEFAULT_BLOB_EXTENT_LOKI_DB_PATH,
   DEFAULT_BLOB_LOKI_DB_PATH,
   DEFAULT_BLOB_PERSISTENCE_ARRAY
 } from "./utils/constants";
+import { StoreDestinationArray } from "../common/persistence/IExtentStore";
 
 export class BlobServerFactory {
   public async createServer(
     blobEnvironment?: IBlobEnvironment
+  ): Promise<BlobServer | SqlBlobServer> {
+    return this.createActualServer(
+      blobEnvironment, 
+      DEFAULT_BLOB_PERSISTENCE_ARRAY, 
+      DEFAULT_BLOB_PERSISTENCE_PATH,
+      "AZURITE_DB",
+      DEFAULT_BLOB_SERVER_HOST_NAME,
+      DEFAULT_BLOB_LISTENING_PORT,
+      DEFAULT_BLOB_LOKI_DB_PATH,
+      DEFAULT_BLOB_EXTENT_LOKI_DB_PATH,
+      SqlBlobServer,
+      BlobServer
+    );
+  }
+
+  protected async createActualServer(
+    blobEnvironment: IBlobEnvironment | undefined,
+    persistenceArray: StoreDestinationArray,
+    persistencePath: string,
+    dbKey: string,
+    defaultHost: string,
+    defaultPort: number,
+    defaultLokiDBPath: string,
+    defaultExtentLokiDBPath: string,
+    sqlSeverClass: any,
+    blobServerClass: any,
   ): Promise<BlobServer | SqlBlobServer> {
     // TODO: Check it's in Visual Studio Code environment or not
     const isVSC = false;
@@ -32,22 +59,22 @@ export class BlobServerFactory {
         );
       }
 
-      DEFAULT_BLOB_PERSISTENCE_ARRAY[0].locationPath = join(
+      persistenceArray[0].locationPath = join(
         location,
-        DEFAULT_BLOB_PERSISTENCE_PATH
+        persistencePath
       );
 
       // TODO: Check we need to create blob server against SQL or Loki
-      const databaseConnectionString = process.env.AZURITE_DB;
+      const databaseConnectionString = process.env[dbKey];
       const isSQL = databaseConnectionString !== undefined;
 
       if (isSQL) {
         const config = new SqlBlobConfiguration(
-          env.blobHost(),
-          env.blobPort(),
+          env.blobHost() || defaultHost,
+          env.blobPort() || defaultPort,
           databaseConnectionString!,
           DEFAULT_SQL_OPTIONS,
-          DEFAULT_BLOB_PERSISTENCE_ARRAY,
+          persistenceArray,
           !env.silent(),
           undefined,
           debugFilePath !== undefined,
@@ -61,14 +88,14 @@ export class BlobServerFactory {
           env.disableProductStyleUrl()
         );
 
-        return new SqlBlobServer(config);
+        return new sqlSeverClass(config);
       } else {
         const config = new BlobConfiguration(
-          env.blobHost(),
-          env.blobPort(),
-          join(location, DEFAULT_BLOB_LOKI_DB_PATH),
-          join(location, DEFAULT_BLOB_EXTENT_LOKI_DB_PATH),
-          DEFAULT_BLOB_PERSISTENCE_ARRAY,
+          env.blobHost() || defaultHost,
+          env.blobPort() || defaultPort,
+          join(location, defaultLokiDBPath),
+          join(location, defaultExtentLokiDBPath),
+          persistenceArray,
           !env.silent(),
           undefined,
           debugFilePath !== undefined,
@@ -81,7 +108,7 @@ export class BlobServerFactory {
           env.oauth(),
           env.disableProductStyleUrl()
         );
-        return new BlobServer(config);
+        return new blobServerClass(config);
       }
     } else {
       // TODO: Add BlobServer construction in VSC

src/blob/IBlobEnvironment.ts

@@ -1,6 +1,11 @@
 export default interface IBlobEnvironment {
+  //dfs
+  datalakeHost(): string | undefined;
+  datalakePort(): number | undefined;
+  //blob
   blobHost(): string | undefined;
   blobPort(): number | undefined;
+  //common
   location(): Promise<string>;
   silent(): boolean;
   loose(): boolean;

src/blob/SqlBlobConfiguration.ts

@@ -4,9 +4,7 @@ import ConfigurationBase from "../common/ConfigurationBase";
 import { StoreDestinationArray } from "../common/persistence/IExtentStore";
 import { DEFAULT_SQL_OPTIONS } from "../common/utils/constants";
 import {
-  DEFAULT_BLOB_LISTENING_PORT,
   DEFAULT_BLOB_PERSISTENCE_ARRAY,
-  DEFAULT_BLOB_SERVER_HOST_NAME,
   DEFAULT_ENABLE_ACCESS_LOG,
   DEFAULT_ENABLE_DEBUG_LOG
 } from "./utils/constants";
@@ -20,8 +18,8 @@ import {
  */
 export default class SqlBlobConfiguration extends ConfigurationBase {
   public constructor(
-    host: string = DEFAULT_BLOB_SERVER_HOST_NAME,
-    port: number = DEFAULT_BLOB_LISTENING_PORT,
+    host: string,
+    port: number,
     public readonly sqlURL: string,
     public readonly sequelizeOptions: SequelizeOptions = DEFAULT_SQL_OPTIONS,
     public readonly persistenceArray: StoreDestinationArray = DEFAULT_BLOB_PERSISTENCE_ARRAY,
@@ -35,7 +33,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
     key: string = "",
     pwd: string = "",
     oauth?: string,
-    disableProductStyleUrl: boolean = false
+    disableProductStyleUrl: boolean = false,
+    clearDB: boolean = false
   ) {
     super(
       host,
@@ -50,7 +49,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
       key,
       pwd,
       oauth,
-      disableProductStyleUrl
+      disableProductStyleUrl,
+      clearDB
     );
   }
 }