Skip to content

Commit 237de17

Browse files
seymourisdeadseymourisdead
authored
Backfill for created updated closed (#66)
1 parent a3387e4 commit 237de17

File tree

6 files changed

+324
-3
lines changed

6 files changed

+324
-3
lines changed

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"scripts": {
1515
"build": "tsc",
1616
"start": "probot run ./lib/webhooks.js",
17+
"start:backfill": "probot run ./lib/backfill.js",
1718
"test": "jest --detectOpenHandles"
1819
},
1920
"dependencies": {

‎src/backfill.ts

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import {Probot} from "probot";
2+
3+
import {backfill_created_updated_deleted} from "./modules";
4+
5+
// backfill scripts for automations, mostly run locally
6+
export = async (app: Probot) => {
7+
8+
// @ts-ignore
9+
const octo = await app.auth(process.env.GITHUB_INSTALLATION_ID);
10+
backfill_created_updated_deleted(octo);
11+
};

‎src/modules/backfill_created_updated_deleted/index.ts

+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
import {ProbotOctokit} from "probot";
2+
import {
3+
getProjectItems, setDateField,
4+
} from "../../shared/graphql_queries";
5+
import {logger} from "../../shared/logger";
6+
import {ALL_TEAMS_PROJECTS, NeonProject} from "../../shared/project_ids";
7+
import {isDryRun} from "../../shared/utils";
8+
9+
const config: Array<Pick<NeonProject, 'projectId' | 'projectNumber' | 'updatedAtFieldId' | 'createdAtFieldId' | 'closedAtFieldId'>> = ALL_TEAMS_PROJECTS.map(({projectId, projectNumber, updatedAtFieldId, createdAtFieldId, closedAtFieldId}) => {
10+
if (!projectId || !projectNumber) {
11+
return;
12+
}
13+
14+
if (!createdAtFieldId && !updatedAtFieldId && !closedAtFieldId) {
15+
return;
16+
}
17+
18+
return {
19+
projectNumber,
20+
projectId,
21+
createdAtFieldId,
22+
updatedAtFieldId,
23+
closedAtFieldId,
24+
}
25+
}).filter(pr => Boolean(pr)) as Array<Pick<NeonProject, 'projectId' | 'projectNumber' | 'updatedAtFieldId' | 'createdAtFieldId' | 'closedAtFieldId'>>;
26+
console.log(config)
27+
28+
export const backfill_created_updated_deleted = async (octokit: ProbotOctokit) => {
29+
console.time('backfill_created_updated_deleted')
30+
let graphqlCounter = 0;
31+
let backfilledItems = 0;
32+
for (let project of config) {
33+
if (!project) {
34+
break;
35+
}
36+
console.time('backfill_created_updated_deleted' + project.projectNumber)
37+
let pageInfo: {
38+
hasNextPage: boolean,
39+
endCursor: string,
40+
} = {
41+
hasNextPage: true,
42+
endCursor: '',
43+
}
44+
while (pageInfo.hasNextPage) {
45+
try {
46+
const res: {
47+
search: {
48+
pageInfo: {
49+
endCursor: string,
50+
hasNextPage: boolean,
51+
startCursor: string,
52+
hasPreviousPage: boolean,
53+
}
54+
issueCount: number,
55+
nodes: Array<{
56+
number: number,
57+
title: string,
58+
createdAt: string,
59+
updatedAt: string,
60+
closedAt?: string,
61+
repository: {
62+
name: string,
63+
}
64+
author: {
65+
login: string,
66+
}
67+
projectItems: {
68+
nodes: Array<{
69+
id: string,
70+
isArchived: boolean,
71+
type: 'ISSUE',
72+
updatedAt: string,
73+
project: {
74+
id: string
75+
}
76+
fieldValues: {
77+
nodes: Array<{
78+
field: {
79+
id: string,
80+
name: string,
81+
}
82+
date: string;
83+
}>
84+
},
85+
}>
86+
}
87+
}>
88+
}
89+
} = await octokit.graphql(getProjectItems, {
90+
q: `org:neondatabase project:neondatabase/${project.projectNumber} is:closed`,
91+
cursor: pageInfo.endCursor,
92+
});
93+
logger('info', `total count: ${res.search.issueCount}`);
94+
const { search } = res;
95+
graphqlCounter++;
96+
logger('info', `processing page from cursor ${pageInfo.endCursor}, itemsCount: ${search.nodes.length}`);
97+
98+
99+
pageInfo.endCursor = search.pageInfo.endCursor
100+
pageInfo.hasNextPage = search.pageInfo.hasNextPage
101+
let i = 0;
102+
for (let issue of search.nodes) {
103+
let backfilled = false;
104+
i++;
105+
logger('info', `it. #${i} Processing issue #${issue.number} ${issue.title}`)
106+
if (!issue.projectItems || !issue.projectItems.nodes || !issue.projectItems.nodes.length) {
107+
logger('info', `Skip Processing issue #${issue.number} ${issue.title}: Does not belong to the project ${project.projectNumber}`)
108+
continue;
109+
}
110+
111+
const projectItem = issue.projectItems.nodes.find((conn) => (conn.project.id === project.projectId));
112+
113+
if (!projectItem) {
114+
logger('info', `Skip Processing issue #${issue.number} ${issue.title}: Does not belong to the project ${project.projectNumber}`)
115+
continue;
116+
}
117+
118+
if (projectItem.isArchived) {
119+
logger('info', `Skip Processing issue #${issue.number} ${issue.title}: Archived in project ${project.projectNumber}`)
120+
continue;
121+
}
122+
123+
const createdAtFieldDesiredValue = issue.createdAt;
124+
const updatedAtFiedDesiredValue = (new Date(projectItem.updatedAt).getTime()) >= new Date(issue.updatedAt).getTime() ?
125+
projectItem.updatedAt : issue.updatedAt;
126+
const closedAtFieldDesiredValue = issue.closedAt;
127+
128+
// @ts-ignore
129+
const dateFieldsValues = Object.fromEntries(projectItem.fieldValues.nodes.map((fieldValueObj) => {
130+
if (fieldValueObj.field) {
131+
return [fieldValueObj.field.id, fieldValueObj.date];
132+
}
133+
return;
134+
}).filter(Boolean));
135+
136+
if (project.createdAtFieldId && !dateFieldsValues[project.createdAtFieldId] && createdAtFieldDesiredValue) {
137+
graphqlCounter++;
138+
backfilled = true;
139+
if (!isDryRun()) {
140+
try {
141+
await octokit.graphql(setDateField, {
142+
project_id: project.projectId,
143+
project_item_id: projectItem.id,
144+
date_field_id: project.createdAtFieldId,
145+
value: createdAtFieldDesiredValue
146+
})
147+
} catch(e) {
148+
logger('error', e)
149+
}
150+
}
151+
logger('info', `Set created at for issue #${issue.number} ${issue.title} in project ${project.projectId} with value ${createdAtFieldDesiredValue}`)
152+
}
153+
154+
if (project.updatedAtFieldId && !dateFieldsValues[project.updatedAtFieldId] && updatedAtFiedDesiredValue) {
155+
graphqlCounter++;
156+
backfilled = true;
157+
158+
if (!isDryRun()) {
159+
try {
160+
await octokit.graphql(setDateField, {
161+
project_id: project.projectId,
162+
project_item_id: projectItem.id,
163+
date_field_id: project.updatedAtFieldId,
164+
value: updatedAtFiedDesiredValue
165+
})
166+
} catch(e) {
167+
logger('error', e)
168+
}
169+
}
170+
logger('info', `Set updated at for issue #${issue.number} ${issue.title} in project ${project.projectId} with value ${updatedAtFiedDesiredValue}`)
171+
}
172+
173+
if (project.closedAtFieldId && !dateFieldsValues[project.closedAtFieldId] && closedAtFieldDesiredValue) {
174+
graphqlCounter++;
175+
backfilled = true;
176+
177+
if (!isDryRun()) {
178+
try {
179+
await octokit.graphql(setDateField, {
180+
project_id: project.projectId,
181+
project_item_id: projectItem.id,
182+
date_field_id: project.closedAtFieldId,
183+
value: closedAtFieldDesiredValue
184+
})
185+
} catch (e) {
186+
logger('error', e)
187+
}
188+
}
189+
logger('info', `Set updated at for issue #${issue.number} ${issue.title} in project ${project.projectId} with value ${closedAtFieldDesiredValue}`)
190+
}
191+
if (backfilled) {
192+
backfilledItems++
193+
}
194+
logger('info', `Done processing issue #${issue.number} ${issue.title}`)
195+
}
196+
197+
logger('info', search);
198+
} catch (e) {
199+
logger('error', e)
200+
}
201+
202+
}
203+
console.log(`Backfilled ${backfilledItems} items for project ${project.projectNumber}, GraphQL requests count: ${graphqlCounter}`)
204+
console.timeLog('backfill_created_updated_deleted', {
205+
projectNumber: project.projectNumber
206+
})
207+
console.timeEnd('backfill_created_updated_deleted' + project.projectNumber)
208+
}
209+
210+
console.timeEnd('backfill_created_updated_deleted')
211+
}

‎src/modules/index.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,5 @@ export * from './sync_team_label_with_project'
77
export * from './sync_fields_cross_projects';
88
export * from './sync_created_at';
99
export * from './sync_updated_at'
10-
export * from './sync_closed_at'
10+
export * from './sync_closed_at'
11+
export * from './backfill_created_updated_deleted';

‎src/shared/graphql_queries.ts

+65-1
Original file line numberDiff line numberDiff line change
@@ -216,4 +216,68 @@ export const clearFieldValue = `
216216
projectV2Item { id }
217217
}
218218
}
219-
`
219+
`
220+
221+
export const getProjectItems = `
222+
query ($q: String!, $cursor: String!){
223+
search(query: $q, type: ISSUE, first: 100, after: $cursor){
224+
pageInfo {
225+
endCursor,
226+
hasNextPage,
227+
startCursor,
228+
hasPreviousPage,
229+
}
230+
issueCount
231+
nodes {
232+
... on Issue {
233+
number,
234+
title,
235+
createdAt,
236+
updatedAt,
237+
closedAt,
238+
repository {
239+
... on Repository {
240+
name
241+
}
242+
}
243+
author {
244+
... on Actor {
245+
login
246+
}
247+
}
248+
projectItems(first: 10) {
249+
... on ProjectV2ItemConnection {
250+
nodes{
251+
... on ProjectV2Item {
252+
id,
253+
isArchived,
254+
type,
255+
updatedAt,
256+
project {
257+
... on ProjectV2 {
258+
id
259+
}
260+
}
261+
fieldValues(first: 15) {
262+
nodes {
263+
... on ProjectV2ItemFieldValueCommon {
264+
field {
265+
... on ProjectV2FieldCommon {
266+
id
267+
name
268+
}
269+
}
270+
}
271+
... on ProjectV2ItemFieldDateValue {
272+
date
273+
}
274+
}
275+
},
276+
}
277+
}
278+
}
279+
}
280+
}
281+
}
282+
}
283+
}`

0 commit comments

Comments
 (0)