diff --git a/client/lib/Resource.ts b/client/lib/Resource.ts index c09b233ef..4fa1ca157 100644 --- a/client/lib/Resource.ts +++ b/client/lib/Resource.ts @@ -28,6 +28,7 @@ const propertyKeys: (keyof Resource)[] = [ 'type', 'request', 'response', + 'documentUrl', 'data', 'json', 'text', @@ -46,6 +47,10 @@ export default class Resource { return getState(this).resource.url; } + public get documentUrl(): string { + return getState(this).resource.documentUrl; + } + public get type(): ResourceType { return getState(this).resource.type; } diff --git a/client/lib/WebsocketResource.ts b/client/lib/WebsocketResource.ts index 28eecc14c..4875ec2ee 100644 --- a/client/lib/WebsocketResource.ts +++ b/client/lib/WebsocketResource.ts @@ -51,6 +51,10 @@ export default class WebsocketResource extends AwaitedEventTarget { return getState(this).resource.url; } + public get documentUrl(): string { + return getState(this).resource.documentUrl; + } + public get type(): ResourceType { return 'Websocket'; } diff --git a/core/lib/SessionState.ts b/core/lib/SessionState.ts index d3491d96c..6cd95e949 100644 --- a/core/lib/SessionState.ts +++ b/core/lib/SessionState.ts @@ -365,6 +365,7 @@ export default class SessionState { receivedAtCommandId: this.lastCommand?.id, type: resourceType, isRedirect: !!redirectedToUrl, + documentUrl: resourceEvent.documentUrl, request: { ...request, postData: request.postData?.toString(), diff --git a/core/lib/UserProfile.ts b/core/lib/UserProfile.ts index e9eaa3f9c..e74c4836b 100644 --- a/core/lib/UserProfile.ts +++ b/core/lib/UserProfile.ts @@ -12,7 +12,9 @@ export default class UserProfile { public static async export(session: Session) { const cookies = await session.browserContext.getCookies(); - const storage: IDomStorage = {}; + // start with previous storage + const storage: IDomStorage = session.options.userProfile?.storage ?? {}; + for (const tab of session.tabsById.values()) { const page = tab.puppetPage; diff --git a/core/models/ResourcesTable.ts b/core/models/ResourcesTable.ts index dac7ba723..4fb5f82ce 100644 --- a/core/models/ResourcesTable.ts +++ b/core/models/ResourcesTable.ts @@ -45,6 +45,7 @@ export default class ResourcesTable extends SqliteTable { ['browserLoadFailure', 'TEXT'], ['browserBlockedReason', 'TEXT'], ['browserCanceled', 'INTEGER'], + ['documentUrl', 'TEXT'], ], true, ); @@ -106,6 +107,7 @@ export default class ResourcesTable extends SqliteTable { record.browserLoadFailure, record.browserBlockedReason, record.browserCanceled ? 1 : 0, + record.documentUrl, ]); } @@ -172,6 +174,7 @@ export default class ResourcesTable extends SqliteTable { meta.response?.browserLoadFailure, extras.browserBlockedReason, extras.browserCanceled ? 1 : 0, + meta.documentUrl, ]); } @@ -256,4 +259,5 @@ export interface IResourcesRecord { browserLoadFailure?: string; browserBlockedReason?: string; browserCanceled?: boolean; + documentUrl: string; } diff --git a/core/test/user-profile.test.ts b/core/test/user-profile.test.ts index d42863377..fbad6d2a7 100644 --- a/core/test/user-profile.test.ts +++ b/core/test/user-profile.test.ts @@ -218,6 +218,61 @@ document.querySelector('#session').innerHTML = [session1,session2,session3].join await tab2.close(); }); + it("should keep profile information for sites that aren't loaded in a session", async () => { + const meta = await connection.createSession({ + userProfile: { + cookies: [], + storage: { + [koaServer.baseUrl]: { + indexedDB: [], + localStorage: [ + ['Test1', 'value0'], + ['test2', 'value1'], + ], + sessionStorage: [], + }, + 'https://previousSite.org': { + indexedDB: [], + localStorage: [['test', 'site1.org']], + sessionStorage: [], + }, + 'https://site2.org': { + indexedDB: [], + localStorage: [['test2', 'site2.org']], + sessionStorage: [], + }, + }, + }, + }); + const tab = Session.getTab(meta); + Helpers.needsClosing.push(tab.session); + + koaServer.get('/unloaded', ctx => { + ctx.body = ` +

storage page

+ +`; + }); + + await tab.goto(`${koaServer.baseUrl}/unloaded`); + await tab.waitForLoad('PaintingStable'); + + const profile = await tab.session.exportUserProfile(); + expect(profile.cookies).toHaveLength(0); + expect(profile.storage[koaServer.baseUrl]?.localStorage).toHaveLength(2); + expect(profile.storage[koaServer.baseUrl]?.localStorage.find(x => x[0] === 'Test1')).toEqual([ + 'Test1', + 'value1', + ]); + expect(profile.storage['https://previousSite.org'].localStorage).toEqual([ + ['test', 'site1.org'], + ]); + expect(profile.storage['https://site2.org'].localStorage).toEqual([['test2', 'site2.org']]); + await tab.close(); + }); + it('should not make requests to end sites during profile "install"', async () => { const mitmSpy = jest.spyOn(HttpRequestHandler, 'onRequest'); await connection.createSession({ diff --git a/interfaces/IResourceMeta.ts b/interfaces/IResourceMeta.ts index 0603f5a7a..0ede5921e 100644 --- a/interfaces/IResourceMeta.ts +++ b/interfaces/IResourceMeta.ts @@ -9,6 +9,7 @@ export default interface IResourceMeta { request: IResourceRequest; response?: IResourceResponse; type: ResourceType; + documentUrl: string; isRedirect?: boolean; receivedAtCommandId?: number; seenAtCommandId?: number; diff --git a/mitm/handlers/RequestSession.ts b/mitm/handlers/RequestSession.ts index 9eabfb1a1..2f843e3af 100644 --- a/mitm/handlers/RequestSession.ts +++ b/mitm/handlers/RequestSession.ts @@ -244,6 +244,7 @@ export interface IRequestSessionResponseEvent extends IRequestSessionRequestEven export interface IRequestSessionRequestEvent { id: number; request: IResourceRequest; + documentUrl: string; serverAlpn: string; protocol: string; socketId: number; diff --git a/mitm/lib/MitmRequestContext.ts b/mitm/lib/MitmRequestContext.ts index 904b3aadc..0c5bf10fe 100644 --- a/mitm/lib/MitmRequestContext.ts +++ b/mitm/lib/MitmRequestContext.ts @@ -188,6 +188,7 @@ export default class MitmRequestContext { browserRequestId: ctx.browserRequestId, request, response, + documentUrl: ctx.documentUrl, redirectedToUrl: ctx.redirectedToUrl, wasCached: ctx.cacheHandler?.didProposeCachedResource ?? false, resourceType: ctx.resourceType, diff --git a/plugins/default-human-emulator/index.ts b/plugins/default-human-emulator/index.ts index b0a654fa6..81efe6e24 100644 --- a/plugins/default-human-emulator/index.ts +++ b/plugins/default-human-emulator/index.ts @@ -136,10 +136,11 @@ export default class DefaultHumanEmulator extends HumanEmulator { const { nodeId } = targetRect; - const targetPoint = getRandomRectPoint(targetRect, DefaultHumanEmulator.boxPaddingPercent); + let targetPoint = getRandomRectPoint(targetRect, DefaultHumanEmulator.boxPaddingPercent); const didMoveMouse = await this.moveMouseToPoint(targetPoint, targetRect.width, runFn, helper); if (didMoveMouse) { targetRect = await helper.lookupBoundingRect([nodeId], true, true); + targetPoint = getRandomRectPoint(targetRect, DefaultHumanEmulator.boxPaddingPercent); } if (targetRect.elementTag === 'option') { diff --git a/puppet-chrome/lib/ProtocolError.ts b/puppet-chrome/lib/ProtocolError.ts index 62609716e..7ca8da0c7 100644 --- a/puppet-chrome/lib/ProtocolError.ts +++ b/puppet-chrome/lib/ProtocolError.ts @@ -18,6 +18,7 @@ export default class ProtocolError extends Error { this.name = 'ProtocolError'; this.method = method; this.stack = stack; + this.stack = `${this.name}: ${this.message}\n${stack}`; this.remoteError = remoteError; } } diff --git a/puppet/lib/PuppetLaunchError.ts b/puppet/lib/PuppetLaunchError.ts index 0d04c3247..7108bbbda 100644 --- a/puppet/lib/PuppetLaunchError.ts +++ b/puppet/lib/PuppetLaunchError.ts @@ -5,5 +5,6 @@ export default class PuppetLaunchError extends Error implements IPuppetLaunchErr super(message); this.stack = stack; this.name = 'PuppetLaunchError'; + this.stack = `${this.name}: ${this.message}\n${stack}`; } } diff --git a/website/docs/Advanced/Resource.md b/website/docs/Advanced/Resource.md index 65d3ff1bb..a00a93ecd 100644 --- a/website/docs/Advanced/Resource.md +++ b/website/docs/Advanced/Resource.md @@ -24,6 +24,12 @@ Retrieve the network request used to retrieve this resource. The requested url +### documentUrl + +The document (if applicable) that requested this resource. + +#### **Returns** `string` + #### **Returns** `string` ### type diff --git a/website/src/layouts/partials/Header.vue b/website/src/layouts/partials/Header.vue index 041d0cc25..494839344 100644 --- a/website/src/layouts/partials/Header.vue +++ b/website/src/layouts/partials/Header.vue @@ -18,8 +18,6 @@ | Repository a(href="//github.com/ulixee/secret-agent/issues" rel="noopener noreferrer" target="_blank") | Issues - a(href="//github.com/ulixee/secret-agent/projects/1" rel="noopener noreferrer" target="_blank") - | Roadmap