Addresses review requests

- Brings 1pux-to-csv important types and functions inline - Doesn't try to parse 1pux file unless it matches the extension - Move reading of file to a bit later - Improves "add dependency" command - Adds "remove dependency" command
2021-12-18 10:56:28 +00:00 · 2021-12-18 10:56:28 +00:00 · db18adf7fd
parent feb7a3ce53
commit db18adf7fd
8 changed files with 738 additions and 1077 deletions
--- a/README.md
+++ b/README.md
@ -61,6 +61,8 @@ For more configuration options, see [Configuration](#configuration)
 | `npm run dev`          | Starts backend server and client app in dev mode, which watches for changes in the source files and automatically rebuilds/restarts the corresponding components. |
 | `npm test`             | Run tests.                                                                                                                                                        |

+To add dependencies, you can use `scope=[scope-without-@padloc/] npm run add [package]` and to remove them, run `scope=[scope-without-@padloc/] npm run remove [package]`.
+
 ## Configuration

 | Environment Variable | Default                          | Description                                                                                               |
--- a/package.json
+++ b/package.json
@ -34,6 +34,7 @@
    "repl": "cd packages/server && npm run repl && cd ../..",
    "test": "lerna run test",
    "locale:extract": "lerna run extract --scope '@padloc/locale'",
-    "add": "lerna add"
+    "add": "lerna add $1 --scope=@padloc/$scope",
+    "remove": "rm packages/$scope/package-lock.json && lerna exec \"npm uninstall $1\" --scope=@padloc/$scope"
  }
 }
--- a/packages/app/package-lock.json
+++ b/packages/app/package-lock.json
--- a/packages/app/package.json
+++ b/packages/app/package.json
@ -20,7 +20,6 @@
    "npm": "8.2.0"
  },
  "dependencies": {
-    "1pux-to-csv": "1.1.0",
    "@padloc/core": "4.0.0",
    "@padloc/locale": "4.0.0",
    "@simplewebauthn/browser": "4.0.0",
@ -41,6 +40,7 @@
    "event-target-shim": "6.0.2",
    "http-server": "0.12.3",
    "jsqr": "1.4.0",
+    "jszip": "3.7.1",
    "lit": "2.0.0-rc.2",
    "localforage": "1.9.0",
    "marked": "3.0.4",
--- a/packages/app/src/elements/import-dialog.ts
+++ b/packages/app/src/elements/import-dialog.ts
@ -13,9 +13,9 @@ import { saveFile } from "@padloc/core/src/platform";
 import { stringToBytes } from "@padloc/core/src/encoding";

@customElement("pl-import-dialog")
-export class ImportDialog extends Dialog<string | Uint8Array, void> {
+export class ImportDialog extends Dialog<File, void> {
    @state()
-    private _rawData: string | Uint8Array = "";
+    private _rawData: null | string | ArrayBuffer = "";

    @state()
    private _items: VaultItem[] = [];
@ -35,7 +35,7 @@ export class ImportDialog extends Dialog<string | Uint8Array, void> {
                    id="formatSelect"
                    .options=${imp.supportedFormats}
                    .label=${$l("Format")}
-                    @change=${this._parseString}
+                    @change=${this._parseData}
                    disabled
                ></pl-select>

@ -67,13 +67,24 @@ export class ImportDialog extends Dialog<string | Uint8Array, void> {
        `;
    }

-    async show(input: string | Uint8Array) {
+    async show(file: File) {
        await this.updateComplete;
        const result = super.show();
-        this._rawData = input;
-        this._formatSelect.value = ((await imp.guessFormat(input)) || imp.CSV).value;
-        this._parseString();
-        this._vaultSelect.value = app.mainVault!;
+
+        const reader = new FileReader();
+        reader.onload = async () => {
+            this._rawData = reader.result;
+            this._formatSelect.value = (imp.guessFormat(file, this._rawData) || imp.CSV).value;
+            this._parseData();
+            this._vaultSelect.value = app.mainVault!;
+        };
+
+        if (imp.doesFileRequireReadingAsBinary(file)) {
+            reader.readAsArrayBuffer(file);
+        } else {
+            reader.readAsText(file);
+        }
+
        return result;
    }

@ -88,7 +99,7 @@ Github,"work,coding",https://github.com,john.doe@gmail.com,129lskdf93`)
        );
    }

-    private async _parseString(): Promise<void> {
+    private async _parseData(): Promise<void> {
        const rawStr = this._rawData;

        switch (this._formatSelect.value) {
--- a/packages/app/src/elements/settings-tools.ts
+++ b/packages/app/src/elements/settings-tools.ts
@ -27,17 +27,8 @@ export class SettingsTools extends StateMixin(LitElement) {

    private async _importFile() {
        const file = this._fileInput.files![0];
-        const reader = new FileReader();
-        reader.onload = async () => {
-            await this._importDialog.show(reader.result as string | Uint8Array);
-            this._fileInput.value = "";
-        };
-
-        if (file.name.endsWith('.1pux')) {
-            reader.readAsArrayBuffer(file);
-        } else {
-            reader.readAsText(file);
-        }
+        await this._importDialog.show(file);
+        this._fileInput.value = "";
    }

    private _export() {
--- a/packages/app/src/lib/1pux-parser.ts
+++ b/packages/app/src/lib/1pux-parser.ts
@ -0,0 +1,308 @@
+import { loadAsync } from 'jszip';
+
+export type OnePuxItemDetailsLoginField = {
+  value: string;
+  id: string;
+  name: string;
+  fieldType: 'A' | 'B' | 'C' | 'E' | 'I' | 'N' | 'P' | 'R' | 'S' | 'T' | 'U';
+  designation?: 'username' | 'password';
+};
+
+export type OnePuxItemDetailsSection = {
+  title: string;
+  name: string;
+  fields: [
+    {
+      title: string;
+      id: string;
+      value: {
+        concealed?: string;
+        reference?: string;
+        string?: string;
+        email?: string;
+        phone?: string;
+        url?: string;
+        totp?: string;
+        gender?: string;
+        creditCardType?: string;
+        creditCardNumber?: string;
+        monthYear?: number;
+        date?: number;
+      };
+      indexAtSource: number;
+      guarded: boolean;
+      multiline: boolean;
+      dontGenerate: boolean;
+      inputTraits: {
+        keyboard: string;
+        correction: string;
+        capitalization: string;
+      };
+    },
+  ];
+};
+
+export type OnePuxItemDetailsPasswordHistory = {
+  value: string;
+  time: number;
+};
+
+export type OnePuxItemOverviewUrl = {
+  label: string;
+  url: string;
+};
+
+export type OnePuxItem = {
+  item?: {
+    uuid: string;
+    favIndex: number;
+    createdAt: number;
+    updatedAt: number;
+    trashed: boolean;
+    categoryUuid: string;
+    details: {
+      loginFields: OnePuxItemDetailsLoginField[];
+      notesPlain?: string;
+      sections: OnePuxItemDetailsSection[];
+      passwordHistory: OnePuxItemDetailsPasswordHistory[];
+      documentAttributes?: {
+        fileName: string;
+        documentId: string;
+        decryptedSize: number;
+      };
+    };
+    overview: {
+      subtitle: string;
+      urls?: OnePuxItemOverviewUrl[];
+      title: string;
+      url: string;
+      ps?: number;
+      pbe?: number;
+      pgrng?: boolean;
+      tags?: string[];
+    };
+  };
+  file?: {
+    attrs: {
+      uuid: string;
+      name: string;
+      type: string;
+    };
+    path: string;
+  };
+};
+
+export type OnePuxVault = {
+  attrs: {
+    uuid: string;
+    desc: string;
+    avatar: string;
+    name: string;
+    type: 'P' | 'E' | 'U';
+  };
+  items: OnePuxItem[];
+};
+
+export type OnePuxAccount = {
+  attrs: {
+    accountName: string;
+    name: string;
+    avatar: string;
+    email: string;
+    uuid: string;
+    domain: string;
+  };
+  vaults: OnePuxVault[];
+};
+
+export type OnePuxData = {
+  accounts: OnePuxAccount[];
+};
+
+export type OnePuxAttributes = {
+  version: number;
+  description: string;
+  createdAt: number;
+};
+
+export type OnePuxExport = {
+  attributes: OnePuxAttributes;
+  data: OnePuxData;
+};
+
+export const parse1PuxFile = async (
+  fileContents: string | ArrayBuffer,
+) => {
+  try {
+    const zip = await loadAsync(fileContents);
+
+    const attributesContent = await zip
+      .file('export.attributes')!
+      .async('string');
+    const attributes = JSON.parse(attributesContent);
+    const dataContent = await zip.file('export.data')!.async('string');
+    const data = JSON.parse(dataContent);
+
+    return {
+      attributes,
+      data,
+    } as OnePuxExport;
+  } catch (error) {
+    console.error('Failed to parse .1pux file');
+    throw error;
+  }
+};
+
+type RowData = {
+  name: string;
+  tags: string;
+  url: string;
+  username: string;
+  password: string;
+  notes: string;
+  extraFields: ExtraField[];
+};
+
+type ExtraFieldType =
+  | 'username'
+  | 'password'
+  | 'url'
+  | 'email'
+  | 'date'
+  | 'month'
+  | 'credit'
+  | 'phone'
+  | 'totp'
+  | 'text';
+
+type ExtraField = { name: string; value: string; type: ExtraFieldType };
+
+type ParseFieldTypeToExtraFieldType = (
+  field: OnePuxItemDetailsLoginField,
+) => ExtraFieldType;
+
+const parseFieldTypeToExtraFieldType: ParseFieldTypeToExtraFieldType = (
+  field,
+) => {
+  if (field.designation === 'username') {
+    return 'username';
+  } else if (field.designation === 'password') {
+    return 'password';
+  } else if (field.fieldType === 'E') {
+    return 'email';
+  } else if (field.fieldType === 'U') {
+    return 'url';
+  }
+  return 'text';
+};
+
+export const parseToRowData = (
+  item: OnePuxItem['item'],
+  defaultTags?: string[],
+) => {
+  if (!item) {
+    return;
+  }
+
+  const rowData: RowData = {
+    name: item.overview.title,
+    tags: [...(defaultTags || []), ...(item.overview.tags || [])].join(','),
+    url: item.overview.url || '',
+    username: '',
+    password: '',
+    notes: item.details.notesPlain || '',
+    extraFields: [],
+  };
+
+  // Skip documents
+  if (
+    item.details.documentAttributes &&
+    item.details.loginFields.length === 0
+  ) {
+    return;
+  }
+
+  // Extract username, password, and some extraFields
+  item.details.loginFields.forEach((field) => {
+    if (field.designation === 'username') {
+      rowData.username = field.value;
+    } else if (field.designation === 'password') {
+      rowData.password = field.value;
+    } else if (
+      field.fieldType === 'I' ||
+      field.fieldType === 'C' ||
+      field.id.includes(';opid=__') ||
+      field.value === ''
+    ) {
+      // Skip these noisy form-fields
+      return;
+    } else {
+      rowData.extraFields.push({
+        name: field.name || field.id,
+        value: field.value,
+        type: parseFieldTypeToExtraFieldType(field),
+      });
+    }
+  });
+
+  // Extract some more extraFields
+  item.details.sections.forEach((section) => {
+    section.fields.forEach((field) => {
+      let value = '';
+      let type: ExtraFieldType = 'text';
+
+      if (Object.prototype.hasOwnProperty.call(field.value, 'concealed')) {
+        value = field.value.concealed || '';
+      } else if (
+        Object.prototype.hasOwnProperty.call(field.value, 'reference')
+      ) {
+        value = field.value.reference || '';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'string')) {
+        value = field.value.string || '';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'email')) {
+        value = field.value.email || '';
+        type = 'email';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'phone')) {
+        value = field.value.phone || '';
+        type = 'phone';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'url')) {
+        value = field.value.url || '';
+        type = 'url';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'totp')) {
+        value = field.value.totp || '';
+        type = 'totp';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'gender')) {
+        value = field.value.gender || '';
+      } else if (
+        Object.prototype.hasOwnProperty.call(field.value, 'creditCardType')
+      ) {
+        value = field.value.creditCardType || '';
+      } else if (
+        Object.prototype.hasOwnProperty.call(field.value, 'creditCardNumber')
+      ) {
+        value = field.value.creditCardNumber || '';
+        type = 'credit';
+      } else if (
+        Object.prototype.hasOwnProperty.call(field.value, 'monthYear')
+      ) {
+        value =
+          (field.value.monthYear && field.value.monthYear.toString()) || '';
+        type = 'month';
+      } else if (Object.prototype.hasOwnProperty.call(field.value, 'date')) {
+        value = (field.value.date && field.value.date.toString()) || '';
+        type = 'date';
+      } else {
+        // Default, so no data is lost when something new comes up
+        value = JSON.stringify(field.value);
+      }
+
+      rowData.extraFields.push({
+        name: field.title || field.id,
+        value,
+        type,
+      });
+    });
+  });
+
+  return rowData;
+};
--- a/packages/app/src/lib/import.ts
+++ b/packages/app/src/lib/import.ts
@ -1,5 +1,3 @@
-import { parse1PuxFile, parseToRowData } from "1pux-to-csv";
-import { OnePuxItem } from "1pux-to-csv/types";
 import { unmarshal, bytesToString } from "@padloc/core/src/encoding";
 import { PBES2Container } from "@padloc/core/src/container";
 import { validateLegacyContainer, parseLegacyContainer } from "@padloc/core/src/legacy";
@ -8,6 +6,8 @@ import { Err, ErrorCode } from "@padloc/core/src/error";
 import { uuid } from "@padloc/core/src/util";
 import { translate as $l } from "@padloc/locale/src/translate";

+import { parse1PuxFile, parseToRowData, OnePuxItem } from "./1pux-parser";
+
 export interface ImportFormat {
    value: "csv" | "padlock-legacy" | "lastpass" | "padloc" | "1pux";
    label: string;
@ -298,8 +298,7 @@ async function parse1PuxItem(accountName: string, vaultName: string, item: OnePu
                // Do nothing
            }
        } else {
-            // @ts-ignore All of extraField.type possibilities match FieldType.*
-            fields.push(new Field({ name: extraField.name, value: extraField.value, type: extraField.type }));
+            fields.push(new Field({ name: extraField.name, value: extraField.value, type: extraField.type as FieldType }));
        }

    }
@ -307,9 +306,13 @@ async function parse1PuxItem(accountName: string, vaultName: string, item: OnePu
    return createVaultItem(itemName, fields, tags);
 }

-export async function as1Pux(file: string | Uint8Array): Promise<VaultItem[]> {
+export async function as1Pux(data: null | string | ArrayBuffer): Promise<VaultItem[]> {
+    if (!data) {
+        throw new Err(ErrorCode.INVALID_1PUX);
+    }
+    
    try {
-        const dataExport = await parse1PuxFile(file);
+        const dataExport = await parse1PuxFile(data);

        const items = [];
        
@ -333,19 +336,13 @@ export async function as1Pux(file: string | Uint8Array): Promise<VaultItem[]> {
 }

 /**
- * Checks if a given string/Uint8Array represents a 1Password 1pux file
+ * Checks if a given file name ends with .1pux to avoid trying to parse unnecessarily
 */
-export async function is1Pux(file: string | Uint8Array): Promise<boolean> {
-    try {
-        const dataExport = await parse1PuxFile(file);
-        return Boolean(dataExport.attributes && dataExport.data);
-    } catch (error) {
-        // Ignore
-    }
-    return false;
+export function is1Pux(file: File): boolean {
+    return file.name.endsWith('.1pux');
 }

-export async function guessFormat(data: string | Uint8Array): Promise<ImportFormat | null> {
+export function guessFormat(file: File, data: string | null | ArrayBuffer): ImportFormat {
    if (isPBES2Container(data as string)) {
        return PBES2;
    }
@ -355,9 +352,17 @@ export async function guessFormat(data: string | Uint8Array): Promise<ImportForm
    if (isLastPass(data as string)) {
        return LASTPASS;
    }
-    if (await is1Pux(data)) {
+    if (is1Pux(file)) {
        return ONEPUX;
    }
    
    return CSV;
 }
+
+export function doesFileRequireReadingAsBinary(file: File): boolean {
+    if (is1Pux(file)) {
+        return true;
+    }
+
+    return false;
+}