Merge pull request #50 from mattwebbio/updateGravity

Update gravity after sync

CHANGELOG.md

@@ -4,8 +4,16 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v1.2.2-beta.0](https://github.com/mattwebbio/orbital-sync/compare/v1.2.1...v1.2.2-beta.0)
+
+- Add gravity update [`3b4e4d9`](https://github.com/mattwebbio/orbital-sync/commit/3b4e4d9b540fc416ad1b067b44767538d1f9a485)
+- Make client error wording consistent [`1c3be22`](https://github.com/mattwebbio/orbital-sync/commit/1c3be22dca01314c28014bf1bffcc5e6546d52ff)
+- Stringify non-string logging [`e84e036`](https://github.com/mattwebbio/orbital-sync/commit/e84e03641c1508ec2764e828198a38deb3043484)
+
 #### [v1.2.1](https://github.com/mattwebbio/orbital-sync/compare/v1.2.0...v1.2.1)
 
+> 30 September 2022
+
 - Bump @typescript-eslint/parser from 5.36.2 to 5.38.1 [`#47`](https://github.com/mattwebbio/orbital-sync/pull/47)
 - Bump nodemailer from 6.7.8 to 6.8.0 [`#48`](https://github.com/mattwebbio/orbital-sync/pull/48)
 - Bump jest and @types/jest [`#49`](https://github.com/mattwebbio/orbital-sync/pull/49)
@@ -16,6 +24,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 - Bump node-html-parser from 6.0.0 to 6.1.1 [`#45`](https://github.com/mattwebbio/orbital-sync/pull/45)
 - Bump @typescript-eslint/eslint-plugin from 5.36.2 to 5.38.1 [`#43`](https://github.com/mattwebbio/orbital-sync/pull/43)
 - Bump @honeybadger-io/js from 4.1.3 to 4.3.1 [`#44`](https://github.com/mattwebbio/orbital-sync/pull/44)
+- Update CHANGELOG.md [`5c2542a`](https://github.com/mattwebbio/orbital-sync/commit/5c2542a6b19e7cca8c20cf6240dc0903c6685390)
 
 #### [v1.2.0](https://github.com/mattwebbio/orbital-sync/compare/v1.1.3-beta.2...v1.2.0)
 

README.md

@@ -87,6 +87,7 @@ It is recommended you run this service with Docker.
 | `SECONDARY_HOST_(#)_BASE_URL` | Yes      | N/A     | `http://192.168.1.3` or `https://pihole2.example.com`           | The base URL of your secondary Pi-hole, including the scheme (HTTP or HTTPS) and port but not including a following slash. Replace `(#)` with a number, starting at `1`, to add multiple secondary Pi-holes. |
 | `SECONDARY_HOST_(#)_PASSWORD` | Yes      | N/A     | `mypassword2`                                                   | The password used to log in to the admin interface.                                                                                                                                                          |
 | `INTERVAL_MINUTES`            | No       | 30      | Any non-zero positive integer, for example `5`, `30`, or `1440` | How long to wait between synchronizations. Defaults to five minutes. Remember that the DNS server on your secondary servers restarts everytime a sync is performed.                                          |
+| `UPDATE_GRAVITY`              | No       | `true`  | `true`/`false`                                                  | Triggers a gravity update after a backup has been uploaded to a secondary Pi-hole. This updates adlists and restarts gravity.                                                                                |
 | `SYNC_WHITELIST`              | No       | `true`  | `true`/`false`                                                  | Copies the whitelist                                                                                                                                                                                         |
 | `SYNC_REGEX_WHITELIST`        | No       | `true`  | `true`/`false`                                                  | Copies the regex whitelist                                                                                                                                                                                   |
 | `SYNC_BLACKLIST`              | No       | `true`  | `true`/`false`                                                  | Copies the blacklist                                                                                                                                                                                         |

docker-compose.test.yml

@@ -18,6 +18,15 @@ services:
       interval: 10s
       timeout: 10s
       retries: 5
+  tertiary:
+    image: pihole/pihole:latest
+    environment:
+      WEBPASSWORD: tertiary_password1
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://tertiary/admin/']
+      interval: 10s
+      timeout: 10s
+      retries: 5
   orbital-sync:
     build:
       context: .
@@ -26,9 +35,14 @@ services:
         condition: service_healthy
       secondary:
         condition: service_healthy
+      tertiary:
+        condition: service_healthy
     environment:
       PRIMARY_HOST_BASE_URL: 'http://primary'
       PRIMARY_HOST_PASSWORD: 'primary_password1'
       SECONDARY_HOST_1_BASE_URL: 'http://secondary'
       SECONDARY_HOST_1_PASSWORD: 'secondary_password1'
+      SECONDARY_HOST_2_BASE_URL: 'http://tertiary'
+      SECONDARY_HOST_2_PASSWORD: 'tertiary_password1'
       RUN_ONCE: 'true'
+      VERBOSE: 'true'

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orbital-sync",
-  "version": "1.2.1",
+  "version": "1.2.2-beta.0",
   "type": "module",
   "main": "dist/index.js",
   "license": "MIT",

src/client.test.ts

@@ -1,10 +1,10 @@
 import { jest } from '@jest/globals';
 import nock from 'nock';
 import { Blob } from 'node-fetch';
-import { ErrorNotification } from './notify';
 import { Client } from './client';
 import type { Host } from './config';
 import { Config } from './config';
+import { ErrorNotification } from './notify';
 
 describe('Client', () => {
   const host: Host = {
@@ -38,7 +38,7 @@ describe('Client', () => {
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
         message:
-          'Error: there was an error logging in to "http://10.0.0.2" - are you able to log in with the configured password?',
+          'There was an error logging in to "http://10.0.0.2" - are you able to log in with the configured password?',
         verbose: {
           host: 'http://10.0.0.2',
           status: 500,
@@ -58,7 +58,7 @@ describe('Client', () => {
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
         message:
-          'Error: no token could be found while logging in to "http://10.0.0.2" - are you able to log in with the configured password?',
+          'No token could be found while logging in to "http://10.0.0.2" - are you able to log in with the configured password?',
         verbose: {
           host: 'http://10.0.0.2',
           innerHtml: ''
@@ -79,7 +79,7 @@ describe('Client', () => {
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
         message:
-          'Error: a token was found but could not be validated while logging in to "http://10.0.0.2" - are you able to log in with the configured password?',
+          'A token was found but could not be validated while logging in to "http://10.0.0.2" - are you able to log in with the configured password?',
         verbose: {
           host: 'http://10.0.0.2',
           token: 'abcdef'
@@ -124,7 +124,7 @@ describe('Client', () => {
 
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
-        message: 'Error: failed to download backup from "http://10.0.0.2".',
+        message: 'Failed to download backup from "http://10.0.0.2".',
         verbose: {
           host: 'http://10.0.0.2',
           status: 500,
@@ -142,7 +142,7 @@ describe('Client', () => {
 
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
-        message: 'Error: failed to download backup from "http://10.0.0.2".',
+        message: 'Failed to download backup from "http://10.0.0.2".',
         verbose: {
           host: 'http://10.0.0.2',
           status: 200,
@@ -199,14 +199,14 @@ describe('Client', () => {
       teleporter.done();
     });
 
-    test('should throw BackupUploadError if response is non-200', async () => {
+    test('should throw error if response is non-200', async () => {
       teleporter.post('/admin/scripts/pi-hole/php/teleporter.php').reply(500);
 
       const expectError = expect(client.uploadBackup(backup)).rejects;
 
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
-        message: 'Error: failed to upload backup to "http://10.0.0.2".',
+        message: 'Failed to upload backup to "http://10.0.0.2".',
         verbose: {
           host: 'http://10.0.0.2',
           status: 500,
@@ -215,14 +215,14 @@ describe('Client', () => {
       });
     });
 
-    test('should throw BackupUploadError if response does not end with "OK"', async () => {
+    test('should throw error if response does not end with "OK"', async () => {
       teleporter.post('/admin/scripts/pi-hole/php/teleporter.php').reply(200);
 
       const expectError = expect(client.uploadBackup(backup)).rejects;
 
       await expectError.toBeInstanceOf(ErrorNotification);
       await expectError.toMatchObject({
-        message: 'Error: failed to upload backup to "http://10.0.0.2".',
+        message: 'Failed to upload backup to "http://10.0.0.2".',
         verbose: {
           host: 'http://10.0.0.2',
           status: 200,
@@ -231,8 +231,104 @@ describe('Client', () => {
       });
     });
 
-    test('should upload backup successfully', async () => {
+    test('should throw error if gravity update fails', async () => {
+      teleporter
+        .post('/admin/scripts/pi-hole/php/teleporter.php')
+        .reply(
+          200,
+          'Processed adlist (14 entries)<br>\n' +
+            'Processed adlist group assignments (13 entries)<br>\n' +
+            'Processed blacklist (exact) (0 entries)<br>\n' +
+            'Processed blacklist (regex) (3 entries)<br>\n' +
+            'Processed client (8 entries)<br>\n' +
+            'Processed client group assignments (16 entries)<br>\n' +
+            'Processed local DNS records (41 entries)<br>\n' +
+            'Processed domain_audit (0 entries)<br>\n' +
+            'Processed black-/whitelist group assignments (10 entries)<br>\n' +
+            'Processed group (3 entries)<br>\n' +
+            'Processed whitelist (exact) (4 entries)<br>\n' +
+            'Processed whitelist (regex) (0 entries)<br>\n' +
+            'OK'
+        );
+      teleporter
+        .get('/admin/scripts/pi-hole/php/gravity.sh.php', undefined)
+        .reply(200, '\ndata: \n\ndata:      [✓] TCP (IPv6)\ndata: \ndata: \n\ndata:');
+
+      const expectError = expect(client.uploadBackup(backup)).rejects;
+
+      await expectError.toBeInstanceOf(ErrorNotification);
+      await expectError.toMatchObject({
+        message: 'Failed updating gravity on "http://10.0.0.2".',
+        verbose: {
+          host: 'http://10.0.0.2',
+          status: 200,
+          eventStream: '[✓] TCP (IPv6)'
+        }
+      });
+    });
+
+    test('should upload backup and update gravity successfully', async () => {
+      const syncOptions = jest.spyOn(Config, 'syncOptions', 'get');
+
+      let requestBody = '';
+      teleporter
+        .post('/admin/scripts/pi-hole/php/teleporter.php', (body) => (requestBody = body))
+        .reply(
+          200,
+          'Processed adlist (14 entries)<br>\n' +
+            'Processed adlist group assignments (13 entries)<br>\n' +
+            'Processed blacklist (exact) (0 entries)<br>\n' +
+            'Processed blacklist (regex) (3 entries)<br>\n' +
+            'Processed client (8 entries)<br>\n' +
+            'Processed client group assignments (16 entries)<br>\n' +
+            'Processed local DNS records (41 entries)<br>\n' +
+            'Processed domain_audit (0 entries)<br>\n' +
+            'Processed black-/whitelist group assignments (10 entries)<br>\n' +
+            'Processed group (3 entries)<br>\n' +
+            'Processed whitelist (exact) (4 entries)<br>\n' +
+            'Processed whitelist (regex) (0 entries)<br>\n' +
+            'OK'
+        );
+      teleporter
+        .get('/admin/scripts/pi-hole/php/gravity.sh.php', undefined)
+        .reply(
+          200,
+          '\ndata: \n\ndata:      [✓] TCP (IPv6)\ndata: \ndata: \n\ndata:   [✓] Pi-hole blocking is enabled\ndata: \n\ndata:'
+        );
+
+      const result = await client.uploadBackup(backup);
+
+      expect(result).toStrictEqual(true);
+      expect(syncOptions).toHaveBeenCalled();
+      expect(requestBody).toContain(
+        'name="token"\r\n\r\nabcdefgijklmnopqrstuvwxyzabcdefgijklmnopqrst'
+      );
+      expect(requestBody).toContain('name="whitelist"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="regex_whitelist"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="blacklist"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="regexlist"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="adlist"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="client"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="group"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="auditlog"\r\n\r\nfalse');
+      expect(requestBody).toContain('name="staticdhcpleases"\r\n\r\nfalse');
+      expect(requestBody).toContain('name="localdnsrecords"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="localcnamerecords"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="flushtables"\r\n\r\ntrue');
+      expect(requestBody).toContain('name="action"\r\n\r\nin');
+      expect(requestBody).toContain(
+        'name="zip_file"; filename="backup.tar.gz"\r\nContent-Type: application/octet-stream'
+      );
+      expect(requestBody.match(/Content-Disposition: form-data; name=/g)).toHaveLength(
+        15
+      );
+    });
+
+    test('should not update gravity if `updateGravity` is disabled', async () => {
       const syncOptions = jest.spyOn(Config, 'syncOptions', 'get');
+      const updateGravity = jest
+        .spyOn(Config, 'updateGravity', 'get')
+        .mockReturnValue(false);
 
       let requestBody = '';
       teleporter
@@ -280,6 +376,7 @@ describe('Client', () => {
       expect(requestBody.match(/Content-Disposition: form-data; name=/g)).toHaveLength(
         15
       );
+      updateGravity.mockRestore();
     });
   });
 });

src/client.ts

@@ -34,7 +34,7 @@ export class Client {
     });
     if (response.status !== 200)
       throw new ErrorNotification({
-        message: `Error: there was an error logging in to "${host.baseUrl}" - are you able to log in with the configured password?`,
+        message: `There was an error logging in to "${host.baseUrl}" - are you able to log in with the configured password?`,
         verbose: {
           host: host.baseUrl,
           status: response.status,
@@ -53,7 +53,7 @@ export class Client {
     const tokenDiv = root.querySelector('#token');
     if (!tokenDiv)
       throw new ErrorNotification({
-        message: `Error: no token could be found while logging in to "${host.baseUrl}" - are you able to log in with the configured password?`,
+        message: `No token could be found while logging in to "${host.baseUrl}" - are you able to log in with the configured password?`,
         verbose: {
           host: host.baseUrl,
           innerHtml: root.innerHTML
@@ -63,7 +63,7 @@ export class Client {
     const token = tokenDiv.innerText;
     if (token.length != 44)
       throw new ErrorNotification({
-        message: `Error: a token was found but could not be validated while logging in to "${host.baseUrl}" - are you able to log in with the configured password?`,
+        message: `A token was found but could not be validated while logging in to "${host.baseUrl}" - are you able to log in with the configured password?`,
         verbose: {
           host: host.baseUrl,
           token: token
@@ -89,7 +89,7 @@ export class Client {
       response.headers.get('content-type') !== 'application/gzip'
     )
       throw new ErrorNotification({
-        message: `Error: failed to download backup from "${this.host.baseUrl}".`,
+        message: `Failed to download backup from "${this.host.baseUrl}".`,
         verbose: {
           host: this.host.baseUrl,
           status: response.status,
@@ -110,26 +110,53 @@ export class Client {
     form.append('action', 'in');
     form.append('zip_file', backup, 'backup.tar.gz');
 
-    const response = await this.fetch(
+    const uploadResponse = await this.fetch(
       `${this.host.baseUrl}/admin/scripts/pi-hole/php/teleporter.php`,
       {
         body: form,
         method: 'POST'
       }
     );
-    const text = await response.text();
-    if (response.status !== 200 || !text.endsWith('OK'))
+    const uploadText = await uploadResponse.text();
+    if (uploadResponse.status !== 200 || !uploadText.endsWith('OK'))
       throw new ErrorNotification({
-        message: `Error: failed to upload backup to "${this.host.baseUrl}".`,
+        message: `Failed to upload backup to "${this.host.baseUrl}".`,
         verbose: {
           host: this.host.baseUrl,
-          status: response.status,
-          responseBody: text
+          status: uploadResponse.status,
+          responseBody: uploadText
         }
       });
 
     Log.info(chalk.green(`✔️ Backup uploaded to ${this.host.baseUrl}!`));
-    Log.verbose(`Result:\n${chalk.blue(text)}`);
+    Log.verbose(`Result:\n${chalk.blue(uploadText)}`);
+
+    if (Config.updateGravity) {
+      Log.info(chalk.yellow(`➡️ Updating gravity on ${this.host.baseUrl}...`));
+      const gravityUpdateResponse = await this.fetch(
+        `${this.host.baseUrl}/admin/scripts/pi-hole/php/gravity.sh.php`,
+        { method: 'GET' }
+      );
+
+      const updateText = (await gravityUpdateResponse.text())
+        .replaceAll('\ndata:', '')
+        .trim();
+      if (
+        gravityUpdateResponse.status !== 200 ||
+        !updateText.endsWith('Pi-hole blocking is enabled')
+      )
+        throw new ErrorNotification({
+          message: `Failed updating gravity on "${this.host.baseUrl}".`,
+          verbose: {
+            host: this.host.baseUrl,
+            status: gravityUpdateResponse.status,
+            eventStream: updateText
+          }
+        });
+
+      Log.info(chalk.green(`✔️ Gravity updated on ${this.host.baseUrl}!`));
+      Log.verbose(`Result:\n${chalk.blue(updateText)}`);
+    }
 
     return true;
   }

src/config.test.ts

@@ -238,6 +238,10 @@ describe('Config', () => {
     });
   });
 
+  describe('updateGravity', () => {
+    testToHaveDefaultAndOverride('updateGravity', true, 'UPDATE_GRAVITY');
+  });
+
   describe('verboseMode', () => {
     testToHaveDefaultAndOverride('verboseMode', false, 'VERBOSE');
   });

src/config.ts

@@ -64,6 +64,10 @@ export class Config {
     return this._syncOptions;
   }
 
+  static get updateGravity(): boolean {
+    return process.env['UPDATE_GRAVITY'] !== 'false';
+  }
+
   static get verboseMode(): boolean {
     return process.env['VERBOSE'] === 'true';
   }