Merge pull request #373 from ulixee/docUrl

Various small bug fixes
ulixee · Nov 13, 2021 · b211ba3 · b211ba3
2 parents a4f681e + b9275e6
commit b211ba3
Show file tree

Hide file tree

Showing 14 changed files with 85 additions and 4 deletions.
diff --git a/client/lib/Resource.ts b/client/lib/Resource.ts
@@ -28,6 +28,7 @@ const propertyKeys: (keyof Resource)[] = [
   'type',
   'request',
   'response',
+  'documentUrl',
   'data',
   'json',
   'text',
@@ -46,6 +47,10 @@ export default class Resource {
     return getState(this).resource.url;
   }
 
+  public get documentUrl(): string {
+    return getState(this).resource.documentUrl;
+  }
+
   public get type(): ResourceType {
     return getState(this).resource.type;
   }

diff --git a/client/lib/WebsocketResource.ts b/client/lib/WebsocketResource.ts
@@ -51,6 +51,10 @@ export default class WebsocketResource extends AwaitedEventTarget<IEventType> {
     return getState(this).resource.url;
   }
 
+  public get documentUrl(): string {
+    return getState(this).resource.documentUrl;
+  }
+
   public get type(): ResourceType {
     return 'Websocket';
   }

diff --git a/core/lib/SessionState.ts b/core/lib/SessionState.ts
@@ -365,6 +365,7 @@ export default class SessionState {
       receivedAtCommandId: this.lastCommand?.id,
       type: resourceType,
       isRedirect: !!redirectedToUrl,
+      documentUrl: resourceEvent.documentUrl,
       request: {
         ...request,
         postData: request.postData?.toString(),

diff --git a/core/lib/UserProfile.ts b/core/lib/UserProfile.ts
@@ -12,7 +12,9 @@ export default class UserProfile {
   public static async export(session: Session) {
     const cookies = await session.browserContext.getCookies();
 
-    const storage: IDomStorage = {};
+    // start with previous storage
+    const storage: IDomStorage = session.options.userProfile?.storage ?? {};
+
     for (const tab of session.tabsById.values()) {
       const page = tab.puppetPage;
 

diff --git a/core/models/ResourcesTable.ts b/core/models/ResourcesTable.ts
@@ -45,6 +45,7 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
         ['browserLoadFailure', 'TEXT'],
         ['browserBlockedReason', 'TEXT'],
         ['browserCanceled', 'INTEGER'],
+        ['documentUrl', 'TEXT'],
       ],
       true,
     );
@@ -106,6 +107,7 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
       record.browserLoadFailure,
       record.browserBlockedReason,
       record.browserCanceled ? 1 : 0,
+      record.documentUrl,
     ]);
   }
 
@@ -172,6 +174,7 @@ export default class ResourcesTable extends SqliteTable<IResourcesRecord> {
       meta.response?.browserLoadFailure,
       extras.browserBlockedReason,
       extras.browserCanceled ? 1 : 0,
+      meta.documentUrl,
     ]);
   }
 
@@ -256,4 +259,5 @@ export interface IResourcesRecord {
   browserLoadFailure?: string;
   browserBlockedReason?: string;
   browserCanceled?: boolean;
+  documentUrl: string;
 }
diff --git a/core/test/user-profile.test.ts b/core/test/user-profile.test.ts
@@ -218,6 +218,61 @@ document.querySelector('#session').innerHTML = [session1,session2,session3].join
     await tab2.close();
   });
 
+  it("should keep profile information for sites that aren't loaded in a session", async () => {
+    const meta = await connection.createSession({
+      userProfile: {
+        cookies: [],
+        storage: {
+          [koaServer.baseUrl]: {
+            indexedDB: [],
+            localStorage: [
+              ['Test1', 'value0'],
+              ['test2', 'value1'],
+            ],
+            sessionStorage: [],
+          },
+          'https://previousSite.org': {
+            indexedDB: [],
+            localStorage: [['test', 'site1.org']],
+            sessionStorage: [],
+          },
+          'https://site2.org': {
+            indexedDB: [],
+            localStorage: [['test2', 'site2.org']],
+            sessionStorage: [],
+          },
+        },
+      },
+    });
+    const tab = Session.getTab(meta);
+    Helpers.needsClosing.push(tab.session);
+
+    koaServer.get('/unloaded', ctx => {
+      ctx.body = `<body>
+<h1>storage page</h1>
+<script>
+localStorage.setItem('Test1', 'value1');
+</script>
+</body>`;
+    });
+
+    await tab.goto(`${koaServer.baseUrl}/unloaded`);
+    await tab.waitForLoad('PaintingStable');
+
+    const profile = await tab.session.exportUserProfile();
+    expect(profile.cookies).toHaveLength(0);
+    expect(profile.storage[koaServer.baseUrl]?.localStorage).toHaveLength(2);
+    expect(profile.storage[koaServer.baseUrl]?.localStorage.find(x => x[0] === 'Test1')).toEqual([
+      'Test1',
+      'value1',
+    ]);
+    expect(profile.storage['https://previousSite.org'].localStorage).toEqual([
+      ['test', 'site1.org'],
+    ]);
+    expect(profile.storage['https://site2.org'].localStorage).toEqual([['test2', 'site2.org']]);
+    await tab.close();
+  });
+
   it('should not make requests to end sites during profile "install"', async () => {
     const mitmSpy = jest.spyOn(HttpRequestHandler, 'onRequest');
     await connection.createSession({

diff --git a/interfaces/IResourceMeta.ts b/interfaces/IResourceMeta.ts
@@ -9,6 +9,7 @@ export default interface IResourceMeta {
   request: IResourceRequest;
   response?: IResourceResponse;
   type: ResourceType;
+  documentUrl: string;
   isRedirect?: boolean;
   receivedAtCommandId?: number;
   seenAtCommandId?: number;

diff --git a/mitm/handlers/RequestSession.ts b/mitm/handlers/RequestSession.ts
@@ -244,6 +244,7 @@ export interface IRequestSessionResponseEvent extends IRequestSessionRequestEven
 export interface IRequestSessionRequestEvent {
   id: number;
   request: IResourceRequest;
+  documentUrl: string;
   serverAlpn: string;
   protocol: string;
   socketId: number;

diff --git a/mitm/lib/MitmRequestContext.ts b/mitm/lib/MitmRequestContext.ts
@@ -188,6 +188,7 @@ export default class MitmRequestContext {
       browserRequestId: ctx.browserRequestId,
       request,
       response,
+      documentUrl: ctx.documentUrl,
       redirectedToUrl: ctx.redirectedToUrl,
       wasCached: ctx.cacheHandler?.didProposeCachedResource ?? false,
       resourceType: ctx.resourceType,

diff --git a/plugins/default-human-emulator/index.ts b/plugins/default-human-emulator/index.ts
@@ -136,10 +136,11 @@ export default class DefaultHumanEmulator extends HumanEmulator {
 
     const { nodeId } = targetRect;
 
-    const targetPoint = getRandomRectPoint(targetRect, DefaultHumanEmulator.boxPaddingPercent);
+    let targetPoint = getRandomRectPoint(targetRect, DefaultHumanEmulator.boxPaddingPercent);
     const didMoveMouse = await this.moveMouseToPoint(targetPoint, targetRect.width, runFn, helper);
     if (didMoveMouse) {
       targetRect = await helper.lookupBoundingRect([nodeId], true, true);
+      targetPoint = getRandomRectPoint(targetRect, DefaultHumanEmulator.boxPaddingPercent);
     }
 
     if (targetRect.elementTag === 'option') {

diff --git a/puppet-chrome/lib/ProtocolError.ts b/puppet-chrome/lib/ProtocolError.ts
@@ -18,6 +18,7 @@ export default class ProtocolError extends Error {
     this.name = 'ProtocolError';
     this.method = method;
     this.stack = stack;
+    this.stack = `${this.name}: ${this.message}\n${stack}`;
     this.remoteError = remoteError;
   }
 }
diff --git a/puppet/lib/PuppetLaunchError.ts b/puppet/lib/PuppetLaunchError.ts
@@ -5,5 +5,6 @@ export default class PuppetLaunchError extends Error implements IPuppetLaunchErr
     super(message);
     this.stack = stack;
     this.name = 'PuppetLaunchError';
+    this.stack = `${this.name}: ${this.message}\n${stack}`;
   }
 }
diff --git a/website/docs/Advanced/Resource.md b/website/docs/Advanced/Resource.md
@@ -24,6 +24,12 @@ Retrieve the network request used to retrieve this resource.
 
 The requested url
 
+### documentUrl
+
+The document (if applicable) that requested this resource.
+
+#### **Returns** `string`
+
 #### **Returns** `string`
 
 ### type

diff --git a/website/src/layouts/partials/Header.vue b/website/src/layouts/partials/Header.vue
@@ -18,8 +18,6 @@
               | Repository
             a(href="//github.com/ulixee/secret-agent/issues" rel="noopener noreferrer" target="_blank")
               | Issues
-            a(href="//github.com/ulixee/secret-agent/projects/1" rel="noopener noreferrer" target="_blank")
-              | Roadmap
 </template>
 
 <static-query>