-
Notifications
You must be signed in to change notification settings - Fork 445
Expand file tree
/
Copy pathdatabase-upload.ts
More file actions
157 lines (144 loc) · 5.53 KB
/
database-upload.ts
File metadata and controls
157 lines (144 loc) · 5.53 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
import * as fs from "fs";
import * as actionsUtil from "./actions-util";
import { AnalysisKind } from "./analyses";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
/** Information about a database upload. */
export interface DatabaseUploadResult {
/** Language of the database. */
language: string;
/** Size of the zipped database in bytes. */
zipped_upload_size_bytes?: number;
/** Whether the uploaded database is an overlay base. */
is_overlay_base?: boolean;
/** Time taken to upload database in milliseconds. */
upload_duration_ms?: number;
/** If there was an error during database upload, this is its message. */
error?: string;
}
export async function cleanupAndUploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<DatabaseUploadResult[]> {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return [];
}
if (!config.analysisKinds.includes(AnalysisKind.CodeScanning)) {
logger.debug(
`Not uploading database because 'analysis-kinds: ${AnalysisKind.CodeScanning}' is not enabled.`,
);
return [];
}
if (util.isInTestMode()) {
logger.debug("In test mode. Skipping database upload.");
return [];
}
// Do nothing when not running against github.com
if (
config.gitHubVersion.type !== util.GitHubVariant.DOTCOM &&
config.gitHubVersion.type !== util.GitHubVariant.GHEC_DR
) {
logger.debug("Not running against github.com or GHEC-DR. Skipping upload.");
return [];
}
if (!(await gitUtils.isAnalyzingDefaultBranch())) {
// We only want to upload a database if we are analyzing the default branch.
logger.debug("Not analyzing default branch. Skipping upload.");
return [];
}
// If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages.
const shouldUploadOverlayBase =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi, codeql));
const cleanupLevel = shouldUploadOverlayBase
? CleanupLevel.Overlay
: CleanupLevel.Clear;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, cleanupLevel);
});
const client = getApiClient();
const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;
// Octokit expects the baseUrl to not have a trailing slash,
// but it is included by default in a URL.
let uploadsBaseUrl = uploadsUrl.toString();
if (uploadsBaseUrl.endsWith("/")) {
uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
}
const reports: DatabaseUploadResult[] = [];
for (const language of config.languages) {
let bundledDbSize: number | undefined = undefined;
try {
// Upload the database bundle.
// Although we are uploading arbitrary file contents to the API, it's worth
// noting that it's the API's job to validate that the contents is acceptable.
// This API method is available to anyone with write access to the repo.
const bundledDb = await bundleDb(config, language, codeql, language, {
includeDiagnostics: false,
});
bundledDbSize = fs.statSync(bundledDb).size;
const bundledDbReadStream = fs.createReadStream(bundledDb);
const commitOid = await gitUtils.getCommitOid(
actionsUtil.getRequiredInput("checkout_path"),
);
try {
const startTime = performance.now();
await client.request(
`POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
{
baseUrl: uploadsBaseUrl,
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
language,
name: `${language}-database`,
commit_oid: commitOid,
data: bundledDbReadStream,
headers: {
authorization: `token ${apiDetails.auth}`,
"Content-Type": "application/zip",
"Content-Length": bundledDbSize,
},
},
);
const endTime = performance.now();
reports.push({
language,
zipped_upload_size_bytes: bundledDbSize,
is_overlay_base: shouldUploadOverlayBase,
upload_duration_ms: endTime - startTime,
});
logger.debug(`Successfully uploaded database for ${language}`);
} finally {
bundledDbReadStream.close();
}
} catch (e) {
// Log a warning but don't fail the workflow
logger.warning(
`Failed to upload database for ${language}: ${util.getErrorMessage(e)}`,
);
reports.push({
language,
error: util.getErrorMessage(e),
...(bundledDbSize !== undefined
? { zipped_upload_size_bytes: bundledDbSize }
: {}),
});
}
}
return reports;
}