Skip to content

Commit

Permalink
BambooHR: Handle unsupported date formats (#592)
Browse files Browse the repository at this point in the history 
* BambooHR: Handle invalid dates

* Fix prev commit

* Fix log message
  • Loading branch information
@cjwooo
cjwooo authored Jul 28, 2022
1 parent 820b0d8 commit bf4cf45
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 13 deletions.
30 changes: 23 additions & 7 deletions destinations/airbyte-faros-destination/src/converters/bamboohr/users.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {AirbyteRecord} from 'faros-airbyte-cdk';
import {AirbyteLogger, AirbyteRecord} from 'faros-airbyte-cdk';
import {Utils} from 'faros-feeds-sdk';
import {intersection, uniq} from 'lodash';

Expand All @@ -7,6 +7,8 @@ import {BambooHRConverter} from './common';
import {User} from './models';

export class Users extends BambooHRConverter {
private logger = new AirbyteLogger();

readonly destinationModels: ReadonlyArray<DestinationModel> = [
'geo_Address',
'geo_Location',
Expand All @@ -27,13 +29,27 @@ export class Users extends BambooHRConverter {
): Promise<ReadonlyArray<DestinationRecord>> {
const source = this.streamName.source;
const user = record.record.data as User;
const joinedAt = Utils.toDate(user.hireDate);
const terminatedAt =
user.terminationDate == '0000-00-00'
? null
: Utils.toDate(user.terminationDate);
const manager = user.supervisorEId ? {uid: user.supervisorEId} : undefined;
const uid = user.id;
let joinedAt = Utils.toDate(user.hireDate);
if (isNaN(joinedAt?.getTime())) {
this.logger.warn(
`Found unexpected hire date ${user.hireDate} for user id ${user.id}`
);
joinedAt = null;
}
let terminatedAt: Date;
if (user.terminationDate === '0000-00-00') {
terminatedAt = null;
} else {
terminatedAt = Utils.toDate(user.terminationDate);
if (isNaN(terminatedAt?.getTime())) {
this.logger.warn(
`Found unexpected termination date ${user.terminationDate} for user id ${user.id}`
);
terminatedAt = null;
}
}
const manager = user.supervisorEId ? {uid: user.supervisorEId} : undefined;
const res: DestinationRecord[] = [];

if (this.bootstrapTeamsFromManagers(ctx)) {
Expand Down
12 changes: 6 additions & 6 deletions destinations/airbyte-faros-destination/test/converters/bamboohr.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ describe('bamboohr', () => {
logger.debug(stdout);

const processedByStream = {
users: 86,
users: 87,
};
const processed = _(processedByStream)
.toPairs()
Expand All @@ -59,13 +59,13 @@ describe('bamboohr', () => {
.value();

const writtenByModel = {
geo_Address: 86,
geo_Location: 86,
identity_Identity: 86,
geo_Address: 87,
geo_Location: 87,
identity_Identity: 87,
org_Department: 9,
org_Employee: 86,
org_Employee: 87,
org_Team: 26,
org_TeamMembership: 111,
org_TeamMembership: 112,
};

const processedTotal = _(processedByStream).values().sum();
Expand Down
1 change: 1 addition & 0 deletions destinations/airbyte-faros-destination/test/resources/bamboohr/all-streams.log
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{"record":{"stream":"mytestsource__bamboohr__users","emitted_at":1653145791152,"data":{"id":"1","acaStatus":null,"address1":"815 Heather St","address2":null,"age":"30","bestEmail":"[email protected]","birthday":"04-15","city":"Vancouver","country":"Canada","createdByUserId":null,"dateOfBirth":"1992-04-15","department":"Human Resources","division":"North America","eeo":"Technicians","employeeNumber":"59","employmentHistoryStatus":"Contractor","ethnicity":"White","exempt":"Exempt","firstName":"Dwight","flsaCode":"Exempt","fullName1":"Dwight Goodman","fullName2":"Goodman, Dwight","fullName3":"Goodman, Dwight","fullName4":"Goodman, Dwight","fullName5":"Dwight Goodman","displayName":"Dwight Goodman","gender":"Male","hireDate":"Bad Date","originalHireDate":"0000-00-00","homeEmail":null,"homePhone":"604-555-3557","isPhotoUploaded":"false","jobTitle":"Technical Recruiter","lastChanged":"2022-05-21T03:17:36+00:00","lastName":"Goodman","location":"Vancouver, Canada","maritalStatus":"Single","middleName":null,"mobilePhone":"604-555-3557","payChangeReason":null,"payGroupId":null,"payRate":"68000 CAD","payRateEffectiveDate":"2021-09-30","payType":"Salary","paidPer":"Year","paySchedule":"Twice a month","payScheduleId":"18347","payFrequency":"Twice a month","preferredName":null,"ssn":null,"state":"BC","stateCode":"","status":"Active","supervisor":"Anderson, Shannon","supervisorId":"4","supervisorEId":"7","supervisorEmail":"[email protected]","terminationDate":"Bad Date Again","workEmail":"[email protected]","workPhone":"604-555-3134","workPhonePlusExtension":"604-555-3134","workPhoneExtension":null,"zipcode":"V6H 2M5"}},"type":"RECORD"}
{"record":{"stream":"mytestsource__bamboohr__users","emitted_at":1653145791152,"data":{"id":"61","acaStatus":null,"address1":"815 Heather St","address2":null,"age":"30","bestEmail":"[email protected]","birthday":"04-15","city":"Vancouver","country":"Canada","createdByUserId":null,"dateOfBirth":"1992-04-15","department":"Human Resources","division":"North America","eeo":"Technicians","employeeNumber":"59","employmentHistoryStatus":"Contractor","ethnicity":"White","exempt":"Exempt","firstName":"Dwight","flsaCode":"Exempt","fullName1":"Dwight Goodman","fullName2":"Goodman, Dwight","fullName3":"Goodman, Dwight","fullName4":"Goodman, Dwight","fullName5":"Dwight Goodman","displayName":"Dwight Goodman","gender":"Male","hireDate":"2021-09-30","originalHireDate":"0000-00-00","homeEmail":null,"homePhone":"604-555-3557","isPhotoUploaded":"false","jobTitle":"Technical Recruiter","lastChanged":"2022-05-21T03:17:36+00:00","lastName":"Goodman","location":"Vancouver, Canada","maritalStatus":"Single","middleName":null,"mobilePhone":"604-555-3557","payChangeReason":null,"payGroupId":null,"payRate":"68000 CAD","payRateEffectiveDate":"2021-09-30","payType":"Salary","paidPer":"Year","paySchedule":"Twice a month","payScheduleId":"18347","payFrequency":"Twice a month","preferredName":null,"ssn":null,"state":"BC","stateCode":"","status":"Active","supervisor":"Anderson, Shannon","supervisorId":"4","supervisorEId":"7","supervisorEmail":"[email protected]","terminationDate":"0000-00-00","workEmail":"[email protected]","workPhone":"604-555-3134","workPhonePlusExtension":"604-555-3134","workPhoneExtension":null,"zipcode":"V6H 2M5"}},"type":"RECORD"}
{"record":{"stream":"mytestsource__bamboohr__users","emitted_at":1653145792452,"data":{"id":"75","acaStatus":null,"address1":"12 Market St","address2":null,"age":"33","bestEmail":"[email protected]","birthday":"01-13","city":"Sydney","country":"Australia","createdByUserId":null,"dateOfBirth":"1989-01-13","department":"Human Resources","division":"Asia-Pacific","eeo":"Technicians","employeeNumber":"73","employmentHistoryStatus":"Full-Time","ethnicity":"White","exempt":"Non-exempt","firstName":"Adam","flsaCode":"Non-exempt","fullName1":"Adam Holt","fullName2":"Holt, Adam","fullName3":"Holt, Adam","fullName4":"Holt, Adam","fullName5":"Adam Holt","displayName":"Adam Holt","gender":"Male","hireDate":"2021-03-30","originalHireDate":"0000-00-00","homeEmail":null,"homePhone":"+61 2 5555 6498","isPhotoUploaded":"false","jobTitle":"Technical Recruiter","lastChanged":"2022-05-21T03:17:36+00:00","lastName":"Holt","location":"Sydney, Australia","maritalStatus":"Single","middleName":null,"mobilePhone":"+61 2 5555 1984","payChangeReason":null,"payGroupId":null,"payRate":"24 AUD","payRateEffectiveDate":"2021-03-30","payType":"Hourly","paidPer":"Hour","paySchedule":"Every other week","payScheduleId":"18346","payFrequency":"Every other week","preferredName":null,"ssn":null,"state":"NSW","stateCode":"","status":"Active","supervisor":"Agluinda, Christina","supervisorId":"3","supervisorEId":"6","supervisorEmail":"[email protected]","terminationDate":"0000-00-00","workEmail":"[email protected]","workPhone":"+61 2 5555 3105","workPhonePlusExtension":"+61 2 5555 3105","workPhoneExtension":null,"zipcode":"2000"}},"type":"RECORD"}
{"record":{"stream":"mytestsource__bamboohr__users","emitted_at":1653145793484,"data":{"id":"63","acaStatus":null,"address1":"474 Duck Ln","address2":"#301","age":"31","bestEmail":"[email protected]","birthday":"01-12","city":"Bristol","country":"United Kingdom","createdByUserId":null,"dateOfBirth":"1991-01-12","department":"Human Resources","division":"Europe","eeo":"Technicians","employeeNumber":"61","employmentHistoryStatus":"Contractor","ethnicity":"White","exempt":"Exempt","firstName":"Lewis","flsaCode":"Exempt","fullName1":"Lewis Shelton","fullName2":"Shelton, Lewis","fullName3":"Shelton, Lewis","fullName4":"Shelton, Lewis","fullName5":"Lewis Shelton","displayName":"Lewis Shelton","gender":"Male","hireDate":"2021-03-30","originalHireDate":"0000-00-00","homeEmail":null,"homePhone":null,"isPhotoUploaded":"false","jobTitle":"Technical Recruiter","lastChanged":"2022-05-21T03:17:36+00:00","lastName":"Shelton","location":"London, UK","maritalStatus":"Single","middleName":null,"mobilePhone":"+44 207 555 1980","payChangeReason":null,"payGroupId":null,"payRate":"67000 GBP","payRateEffectiveDate":"2021-03-30","payType":"Salary","paidPer":"Year","paySchedule":"Twice a month","payScheduleId":"18347","payFrequency":"Twice a month","preferredName":null,"ssn":null,"state":"","stateCode":"","status":"Active","supervisor":"Adams, Ashley","supervisorId":"2","supervisorEId":"5","supervisorEmail":"[email protected]","terminationDate":"0000-00-00","workEmail":"[email protected]","workPhone":"+44 207 555 4730","workPhonePlusExtension":"+44 207 555 4730 ext 137","workPhoneExtension":"137","zipcode":"BS1 6EG"}},"type":"RECORD"}
Expand Down

0 comments on commit bf4cf45

Please sign in to comment.