diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index bd7f2b7b6..af80372a5 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -394,7 +394,7 @@ }, "fullScopeAllowed": true, "defaultClientScopes": [ - "web-origins", "profile", "roles", "email" + "web-origins", "profile", "roles", "email", "basic" ], "optionalClientScopes": [ "offline_access", "{{ .Values.apps.accounts.admin.role }}-scope" diff --git a/applications/osb-portal/src/apiclient/workspaces/.openapi-generator/FILES b/applications/osb-portal/src/apiclient/workspaces/.openapi-generator/FILES index 892d529ce..4dcc0d8f1 100644 --- a/applications/osb-portal/src/apiclient/workspaces/.openapi-generator/FILES +++ b/applications/osb-portal/src/apiclient/workspaces/.openapi-generator/FILES @@ -1,3 +1,4 @@ +apis/DandiApi.ts apis/K8sApi.ts apis/RestApi.ts apis/index.ts @@ -30,6 +31,12 @@ models/ResourceOrigin.ts models/ResourceStatus.ts models/ResourceType.ts models/Tag.ts +models/UploadFinalizeRequest.ts +models/UploadFinalizeResponse.ts +models/UploadInitRequest.ts +models/UploadInitResponse.ts +models/UploadPart.ts +models/UploadedPart.ts models/User.ts models/Valid.ts models/VolumeStorage.ts diff --git a/applications/osb-portal/src/apiclient/workspaces/apis/DandiApi.ts b/applications/osb-portal/src/apiclient/workspaces/apis/DandiApi.ts new file mode 100644 index 000000000..31fe8d56c --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/apis/DandiApi.ts @@ -0,0 +1,127 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +import * as runtime from '../runtime'; +import { + UploadFinalizeRequest, + UploadFinalizeRequestFromJSON, + UploadFinalizeRequestToJSON, + UploadFinalizeResponse, + UploadFinalizeResponseFromJSON, + UploadFinalizeResponseToJSON, + UploadInitRequest, + UploadInitRequestFromJSON, + UploadInitRequestToJSON, + UploadInitResponse, + UploadInitResponseFromJSON, + UploadInitResponseToJSON, +} from '../models'; + +export interface WorkspacesControllersDandiUploadControllerDandiUploadFinalizeRequest { + uploadFinalizeRequest: UploadFinalizeRequest; +} + +export interface WorkspacesControllersDandiUploadControllerDandiUploadInitRequest { + uploadInitRequest: UploadInitRequest; +} + +/** + * + */ +export class DandiApi extends runtime.BaseAPI { + + /** + * Complete the DANDI upload, create/attach the workspace, and run the protocol script in it. Synchronous — does not return until the script finishes, which can take minutes. + */ + async workspacesControllersDandiUploadControllerDandiUploadFinalizeRaw(requestParameters: WorkspacesControllersDandiUploadControllerDandiUploadFinalizeRequest): Promise> { + if (requestParameters.uploadFinalizeRequest === null || requestParameters.uploadFinalizeRequest === undefined) { + throw new runtime.RequiredError('uploadFinalizeRequest','Required parameter requestParameters.uploadFinalizeRequest was null or undefined when calling workspacesControllersDandiUploadControllerDandiUploadFinalize.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + headerParameters['Content-Type'] = 'application/json'; + + if (this.configuration && this.configuration.accessToken) { + const token = this.configuration.accessToken; + const tokenString = typeof token === 'function' ? token("bearerAuth", []) : token; + + if (tokenString) { + headerParameters["Authorization"] = `Bearer ${tokenString}`; + } + } + const response = await this.request({ + path: `/dandi/upload/finalize`, + method: 'POST', + headers: headerParameters, + query: queryParameters, + body: UploadFinalizeRequestToJSON(requestParameters.uploadFinalizeRequest), + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => UploadFinalizeResponseFromJSON(jsonValue)); + } + + /** + * Complete the DANDI upload, create/attach the workspace, and run the protocol script in it. Synchronous — does not return until the script finishes, which can take minutes. + */ + async workspacesControllersDandiUploadControllerDandiUploadFinalize(requestParameters: WorkspacesControllersDandiUploadControllerDandiUploadFinalizeRequest): Promise { + const response = await this.workspacesControllersDandiUploadControllerDandiUploadFinalizeRaw(requestParameters); + return await response.value(); + } + + /** + * Reserve a DANDI upload. The asset path is derived server-side from the caller\'s identity and is never taken from the request body. + */ + async workspacesControllersDandiUploadControllerDandiUploadInitRaw(requestParameters: WorkspacesControllersDandiUploadControllerDandiUploadInitRequest): Promise> { + if (requestParameters.uploadInitRequest === null || requestParameters.uploadInitRequest === undefined) { + throw new runtime.RequiredError('uploadInitRequest','Required parameter requestParameters.uploadInitRequest was null or undefined when calling workspacesControllersDandiUploadControllerDandiUploadInit.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + headerParameters['Content-Type'] = 'application/json'; + + if (this.configuration && this.configuration.accessToken) { + const token = this.configuration.accessToken; + const tokenString = typeof token === 'function' ? token("bearerAuth", []) : token; + + if (tokenString) { + headerParameters["Authorization"] = `Bearer ${tokenString}`; + } + } + const response = await this.request({ + path: `/dandi/upload/init`, + method: 'POST', + headers: headerParameters, + query: queryParameters, + body: UploadInitRequestToJSON(requestParameters.uploadInitRequest), + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => UploadInitResponseFromJSON(jsonValue)); + } + + /** + * Reserve a DANDI upload. The asset path is derived server-side from the caller\'s identity and is never taken from the request body. + */ + async workspacesControllersDandiUploadControllerDandiUploadInit(requestParameters: WorkspacesControllersDandiUploadControllerDandiUploadInitRequest): Promise { + const response = await this.workspacesControllersDandiUploadControllerDandiUploadInitRaw(requestParameters); + return await response.value(); + } + +} diff --git a/applications/osb-portal/src/apiclient/workspaces/apis/RestApi.ts b/applications/osb-portal/src/apiclient/workspaces/apis/RestApi.ts index 3b0123fb8..db41ed8b6 100644 --- a/applications/osb-portal/src/apiclient/workspaces/apis/RestApi.ts +++ b/applications/osb-portal/src/apiclient/workspaces/apis/RestApi.ts @@ -200,6 +200,10 @@ export interface WorkspacesControllersWorkspaceControllerImportResourcesRequest inlineObject?: InlineObject; } +export interface WorkspacesControllersWorkspaceControllerOpenRequest { + id: number; +} + export interface WorkspacesControllersWorkspaceControllerSetthumbnailRequest { id: number; thumbNail?: Blob; @@ -1484,6 +1488,43 @@ export class RestApi extends runtime.BaseAPI { await this.workspacesControllersWorkspaceControllerImportResourcesRaw(requestParameters); } + /** + * Ensure the workspace volume exists and is ready before opening the workspace. + */ + async workspacesControllersWorkspaceControllerOpenRaw(requestParameters: WorkspacesControllersWorkspaceControllerOpenRequest): Promise> { + if (requestParameters.id === null || requestParameters.id === undefined) { + throw new runtime.RequiredError('id','Required parameter requestParameters.id was null or undefined when calling workspacesControllersWorkspaceControllerOpen.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && this.configuration.accessToken) { + const token = this.configuration.accessToken; + const tokenString = typeof token === 'function' ? token("bearerAuth", []) : token; + + if (tokenString) { + headerParameters["Authorization"] = `Bearer ${tokenString}`; + } + } + const response = await this.request({ + path: `/workspace/{id}/open`.replace(`{${"id"}}`, encodeURIComponent(String(requestParameters.id))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.VoidApiResponse(response); + } + + /** + * Ensure the workspace volume exists and is ready before opening the workspace. + */ + async workspacesControllersWorkspaceControllerOpen(requestParameters: WorkspacesControllersWorkspaceControllerOpenRequest): Promise { + await this.workspacesControllersWorkspaceControllerOpenRaw(requestParameters); + } + /** * Sets the thumbnail of the workspace. */ diff --git a/applications/osb-portal/src/apiclient/workspaces/apis/index.ts b/applications/osb-portal/src/apiclient/workspaces/apis/index.ts index 1d3db4858..e6da88c8a 100644 --- a/applications/osb-portal/src/apiclient/workspaces/apis/index.ts +++ b/applications/osb-portal/src/apiclient/workspaces/apis/index.ts @@ -1,2 +1,3 @@ +export * from './DandiApi'; export * from './K8sApi'; export * from './RestApi'; diff --git a/applications/osb-portal/src/apiclient/workspaces/models/UploadFinalizeRequest.ts b/applications/osb-portal/src/apiclient/workspaces/models/UploadFinalizeRequest.ts new file mode 100644 index 000000000..4e4dd5263 --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/UploadFinalizeRequest.ts @@ -0,0 +1,120 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +import { + UploadedPart, + UploadedPartFromJSON, + UploadedPartFromJSONTyped, + UploadedPartToJSON, +} from './'; + +/** + * + * @export + * @interface UploadFinalizeRequest + */ +export interface UploadFinalizeRequest { + /** + * Omitted on the deduplicated path, where no upload took place. + * @type {string} + * @memberof UploadFinalizeRequest + */ + uploadId?: string; + /** + * + * @type {string} + * @memberof UploadFinalizeRequest + */ + path: string; + /** + * Omitted on the deduplicated path. + * @type {Array} + * @memberof UploadFinalizeRequest + */ + parts?: Array; + /** + * Echoed back from init's deduplicated response; when present, complete/validate are skipped. + * @type {string} + * @memberof UploadFinalizeRequest + */ + blobId?: string; + /** + * Existing workspace to attach to. Omit to create a new one. + * @type {number} + * @memberof UploadFinalizeRequest + */ + workspaceId?: number; + /** + * Name for a new workspace, used only when workspace_id is omitted. + * @type {string} + * @memberof UploadFinalizeRequest + */ + workspaceName?: string; + /** + * Publicly-reachable URL of the protocol's analysis script. Fetched and run server-side inside the workspace; omit to skip running anything. + * @type {string} + * @memberof UploadFinalizeRequest + */ + scriptUrl?: string; + /** + * Filename the script lands under in the workspace. + * @type {string} + * @memberof UploadFinalizeRequest + */ + scriptName?: string; +} + +export function UploadFinalizeRequestFromJSON(json: any): UploadFinalizeRequest { + return UploadFinalizeRequestFromJSONTyped(json, false); +} + +export function UploadFinalizeRequestFromJSONTyped(json: any, ignoreDiscriminator: boolean): UploadFinalizeRequest { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'uploadId': !exists(json, 'upload_id') ? undefined : json['upload_id'], + 'path': json['path'], + 'parts': !exists(json, 'parts') ? undefined : ((json['parts'] as Array).map(UploadedPartFromJSON)), + 'blobId': !exists(json, 'blob_id') ? undefined : json['blob_id'], + 'workspaceId': !exists(json, 'workspace_id') ? undefined : json['workspace_id'], + 'workspaceName': !exists(json, 'workspace_name') ? undefined : json['workspace_name'], + 'scriptUrl': !exists(json, 'script_url') ? undefined : json['script_url'], + 'scriptName': !exists(json, 'script_name') ? undefined : json['script_name'], + }; +} + +export function UploadFinalizeRequestToJSON(value?: UploadFinalizeRequest | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'upload_id': value.uploadId, + 'path': value.path, + 'parts': value.parts === undefined ? undefined : ((value.parts as Array).map(UploadedPartToJSON)), + 'blob_id': value.blobId, + 'workspace_id': value.workspaceId, + 'workspace_name': value.workspaceName, + 'script_url': value.scriptUrl, + 'script_name': value.scriptName, + }; +} + + diff --git a/applications/osb-portal/src/apiclient/workspaces/models/UploadFinalizeResponse.ts b/applications/osb-portal/src/apiclient/workspaces/models/UploadFinalizeResponse.ts new file mode 100644 index 000000000..e15ddeb4f --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/UploadFinalizeResponse.ts @@ -0,0 +1,81 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface UploadFinalizeResponse + */ +export interface UploadFinalizeResponse { + /** + * Path of the registered DANDI asset. + * @type {string} + * @memberof UploadFinalizeResponse + */ + assetPath: string; + /** + * + * @type {string} + * @memberof UploadFinalizeResponse + */ + dandisetUrl?: string; + /** + * + * @type {number} + * @memberof UploadFinalizeResponse + */ + workspaceId: number; + /** + * Everything the protocol script printed, or an explanation of why it did not run. Absent when no script_url was supplied. + * @type {string} + * @memberof UploadFinalizeResponse + */ + scriptOutput?: string; +} + +export function UploadFinalizeResponseFromJSON(json: any): UploadFinalizeResponse { + return UploadFinalizeResponseFromJSONTyped(json, false); +} + +export function UploadFinalizeResponseFromJSONTyped(json: any, ignoreDiscriminator: boolean): UploadFinalizeResponse { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'assetPath': json['asset_path'], + 'dandisetUrl': !exists(json, 'dandiset_url') ? undefined : json['dandiset_url'], + 'workspaceId': json['workspace_id'], + 'scriptOutput': !exists(json, 'script_output') ? undefined : json['script_output'], + }; +} + +export function UploadFinalizeResponseToJSON(value?: UploadFinalizeResponse | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'asset_path': value.assetPath, + 'dandiset_url': value.dandisetUrl, + 'workspace_id': value.workspaceId, + 'script_output': value.scriptOutput, + }; +} + + diff --git a/applications/osb-portal/src/apiclient/workspaces/models/UploadInitRequest.ts b/applications/osb-portal/src/apiclient/workspaces/models/UploadInitRequest.ts new file mode 100644 index 000000000..6d0832bd9 --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/UploadInitRequest.ts @@ -0,0 +1,81 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface UploadInitRequest + */ +export interface UploadInitRequest { + /** + * Slug of the selected protocol; becomes the first segment of the asset path. + * @type {string} + * @memberof UploadInitRequest + */ + taskId?: string; + /** + * + * @type {string} + * @memberof UploadInitRequest + */ + filename: string; + /** + * + * @type {number} + * @memberof UploadInitRequest + */ + size: number; + /** + * S3-multipart digest in DANDI's format (32 hex chars, a dash, the part count), computed in the browser since the bytes never reach this server. + * @type {string} + * @memberof UploadInitRequest + */ + dandiEtag: string; +} + +export function UploadInitRequestFromJSON(json: any): UploadInitRequest { + return UploadInitRequestFromJSONTyped(json, false); +} + +export function UploadInitRequestFromJSONTyped(json: any, ignoreDiscriminator: boolean): UploadInitRequest { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'taskId': !exists(json, 'task_id') ? undefined : json['task_id'], + 'filename': json['filename'], + 'size': json['size'], + 'dandiEtag': json['dandi_etag'], + }; +} + +export function UploadInitRequestToJSON(value?: UploadInitRequest | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'task_id': value.taskId, + 'filename': value.filename, + 'size': value.size, + 'dandi_etag': value.dandiEtag, + }; +} + + diff --git a/applications/osb-portal/src/apiclient/workspaces/models/UploadInitResponse.ts b/applications/osb-portal/src/apiclient/workspaces/models/UploadInitResponse.ts new file mode 100644 index 000000000..eadba720e --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/UploadInitResponse.ts @@ -0,0 +1,88 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +import { + UploadPart, + UploadPartFromJSON, + UploadPartFromJSONTyped, + UploadPartToJSON, +} from './'; + +/** + * + * @export + * @interface UploadInitResponse + */ +export interface UploadInitResponse { + /** + * Absent when the content was already in the archive. + * @type {string} + * @memberof UploadInitResponse + */ + uploadId?: string; + /** + * Asset path derived server-side from the caller's identity. + * @type {string} + * @memberof UploadInitResponse + */ + path?: string; + /** + * + * @type {Array} + * @memberof UploadInitResponse + */ + parts?: Array; + /** + * Set only on the deduplicated path. + * @type {string} + * @memberof UploadInitResponse + */ + blobId?: string; +} + +export function UploadInitResponseFromJSON(json: any): UploadInitResponse { + return UploadInitResponseFromJSONTyped(json, false); +} + +export function UploadInitResponseFromJSONTyped(json: any, ignoreDiscriminator: boolean): UploadInitResponse { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'uploadId': !exists(json, 'upload_id') ? undefined : json['upload_id'], + 'path': !exists(json, 'path') ? undefined : json['path'], + 'parts': !exists(json, 'parts') ? undefined : ((json['parts'] as Array).map(UploadPartFromJSON)), + 'blobId': !exists(json, 'blob_id') ? undefined : json['blob_id'], + }; +} + +export function UploadInitResponseToJSON(value?: UploadInitResponse | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'upload_id': value.uploadId, + 'path': value.path, + 'parts': value.parts === undefined ? undefined : ((value.parts as Array).map(UploadPartToJSON)), + 'blob_id': value.blobId, + }; +} + + diff --git a/applications/osb-portal/src/apiclient/workspaces/models/UploadPart.ts b/applications/osb-portal/src/apiclient/workspaces/models/UploadPart.ts new file mode 100644 index 000000000..189ba5151 --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/UploadPart.ts @@ -0,0 +1,65 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface UploadPart + */ +export interface UploadPart { + /** + * + * @type {number} + * @memberof UploadPart + */ + partNumber?: number; + /** + * Presigned S3 URL the browser PUTs this part's bytes to directly. + * @type {string} + * @memberof UploadPart + */ + url?: string; +} + +export function UploadPartFromJSON(json: any): UploadPart { + return UploadPartFromJSONTyped(json, false); +} + +export function UploadPartFromJSONTyped(json: any, ignoreDiscriminator: boolean): UploadPart { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'partNumber': !exists(json, 'part_number') ? undefined : json['part_number'], + 'url': !exists(json, 'url') ? undefined : json['url'], + }; +} + +export function UploadPartToJSON(value?: UploadPart | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'part_number': value.partNumber, + 'url': value.url, + }; +} + + diff --git a/applications/osb-portal/src/apiclient/workspaces/models/UploadedPart.ts b/applications/osb-portal/src/apiclient/workspaces/models/UploadedPart.ts new file mode 100644 index 000000000..40dde1105 --- /dev/null +++ b/applications/osb-portal/src/apiclient/workspaces/models/UploadedPart.ts @@ -0,0 +1,73 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * OSB Workspaces manager API + * Opensource Brain Platform - Reference Workspaces manager API + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface UploadedPart + */ +export interface UploadedPart { + /** + * + * @type {number} + * @memberof UploadedPart + */ + partNumber?: number; + /** + * + * @type {number} + * @memberof UploadedPart + */ + size?: number; + /** + * The ETag S3 returned for this part's PUT. + * @type {string} + * @memberof UploadedPart + */ + etag?: string; +} + +export function UploadedPartFromJSON(json: any): UploadedPart { + return UploadedPartFromJSONTyped(json, false); +} + +export function UploadedPartFromJSONTyped(json: any, ignoreDiscriminator: boolean): UploadedPart { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'partNumber': !exists(json, 'part_number') ? undefined : json['part_number'], + 'size': !exists(json, 'size') ? undefined : json['size'], + 'etag': !exists(json, 'etag') ? undefined : json['etag'], + }; +} + +export function UploadedPartToJSON(value?: UploadedPart | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'part_number': value.partNumber, + 'size': value.size, + 'etag': value.etag, + }; +} + + diff --git a/applications/osb-portal/src/apiclient/workspaces/models/index.ts b/applications/osb-portal/src/apiclient/workspaces/models/index.ts index d0d39625f..d63161c25 100644 --- a/applications/osb-portal/src/apiclient/workspaces/models/index.ts +++ b/applications/osb-portal/src/apiclient/workspaces/models/index.ts @@ -26,6 +26,12 @@ export * from './ResourceOrigin'; export * from './ResourceStatus'; export * from './ResourceType'; export * from './Tag'; +export * from './UploadFinalizeRequest'; +export * from './UploadFinalizeResponse'; +export * from './UploadInitRequest'; +export * from './UploadInitResponse'; +export * from './UploadPart'; +export * from './UploadedPart'; export * from './User'; export * from './Valid'; export * from './VolumeStorage'; diff --git a/applications/workspaces/api/openapi.yaml b/applications/workspaces/api/openapi.yaml index 034bfcf76..afe754e3d 100644 --- a/applications/workspaces/api/openapi.yaml +++ b/applications/workspaces/api/openapi.yaml @@ -990,8 +990,172 @@ paths: operationId: user_resource_counts summary: Per-user workspace and repository counts (admin only). x-openapi-router-controller: workspaces.controllers.admin_controller + /dandi/upload/init: + post: + requestBody: + description: The file to reserve an upload slot for in EMBER-DANDI. + content: + application/json: + schema: + $ref: '#/components/schemas/UploadInitRequest' + required: true + tags: + - dandi + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UploadInitResponse' + description: >- + Upload reserved. Returns presigned S3 part URLs the browser PUTs to + directly, or (when the content already exists in the archive) an empty + parts list plus the existing blob_id. + '401': + description: Missing or invalid token + security: + - + bearerAuth: [] + operationId: workspaces.controllers.dandi_upload_controller.dandi_upload_init + summary: >- + Reserve a DANDI upload. The asset path is derived server-side from the caller's + identity and is never taken from the request body. + /dandi/upload/finalize: + post: + requestBody: + description: The completed upload to register, attach to a workspace, and analyse. + content: + application/json: + schema: + $ref: '#/components/schemas/UploadFinalizeRequest' + required: true + tags: + - dandi + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UploadFinalizeResponse' + description: Asset registered, workspace attached, script run. + '401': + description: Missing or invalid token + security: + - + bearerAuth: [] + operationId: workspaces.controllers.dandi_upload_controller.dandi_upload_finalize + summary: >- + Complete the DANDI upload, create/attach the workspace, and run the protocol + script in it. Synchronous — does not return until the script finishes, which + can take minutes. components: schemas: + UploadPart: + type: object + properties: + part_number: + type: integer + url: + type: string + description: Presigned S3 URL the browser PUTs this part's bytes to directly. + UploadedPart: + type: object + properties: + part_number: + type: integer + size: + type: integer + etag: + type: string + description: The ETag S3 returned for this part's PUT. + UploadInitRequest: + type: object + required: + - filename + - size + - dandi_etag + properties: + task_id: + type: string + description: >- + Slug of the selected protocol; becomes the first segment of the asset path. + filename: + type: string + size: + type: integer + dandi_etag: + type: string + description: >- + S3-multipart digest in DANDI's format (32 hex chars, a dash, the part + count), computed in the browser since the bytes never reach this server. + UploadInitResponse: + type: object + properties: + upload_id: + type: string + description: Absent when the content was already in the archive. + path: + type: string + description: Asset path derived server-side from the caller's identity. + parts: + type: array + items: + $ref: '#/components/schemas/UploadPart' + blob_id: + type: string + description: Set only on the deduplicated path. + UploadFinalizeRequest: + type: object + required: + - path + properties: + upload_id: + type: string + description: Omitted on the deduplicated path, where no upload took place. + path: + type: string + parts: + type: array + description: Omitted on the deduplicated path. + items: + $ref: '#/components/schemas/UploadedPart' + blob_id: + type: string + description: >- + Echoed back from init's deduplicated response; when present, + complete/validate are skipped. + workspace_id: + type: integer + description: Existing workspace to attach to. Omit to create a new one. + workspace_name: + type: string + description: Name for a new workspace, used only when workspace_id is omitted. + script_url: + type: string + description: >- + Publicly-reachable URL of the protocol's analysis script. Fetched and run + server-side inside the workspace; omit to skip running anything. + script_name: + type: string + description: Filename the script lands under in the workspace. + UploadFinalizeResponse: + type: object + required: + - asset_path + - workspace_id + properties: + asset_path: + type: string + description: Path of the registered DANDI asset. + dandiset_url: + type: string + workspace_id: + type: integer + script_output: + type: string + description: >- + Everything the protocol script printed, or an explanation of why it did + not run. Absent when no script_url was supplied. Valid: type: object properties: @@ -1557,3 +1721,8 @@ tags: - name: Client description: Client + - + name: dandi + description: >- + Brokered uploads into EMBER-DANDI (IDP-43). File bytes go browser -> S3 directly via + presigned part URLs; only metadata passes through this server. diff --git a/applications/workspaces/deploy/values.yaml b/applications/workspaces/deploy/values.yaml index 07895d267..5edda8324 100644 --- a/applications/workspaces/deploy/values.yaml +++ b/applications/workspaces/deploy/values.yaml @@ -54,6 +54,7 @@ harness: secrets: github-user: github-token: + dandi-api-key: "" dependencies: build: - cloudharness-base diff --git a/applications/workspaces/server/.openapi-generator/FILES b/applications/workspaces/server/.openapi-generator/FILES index ca5d3a964..02f72cd35 100644 --- a/applications/workspaces/server/.openapi-generator/FILES +++ b/applications/workspaces/server/.openapi-generator/FILES @@ -33,6 +33,12 @@ workspaces/models/resource_origin.py workspaces/models/resource_status.py workspaces/models/resource_type.py workspaces/models/tag.py +workspaces/models/upload_finalize_request.py +workspaces/models/upload_finalize_response.py +workspaces/models/upload_init_request.py +workspaces/models/upload_init_response.py +workspaces/models/upload_part.py +workspaces/models/uploaded_part.py workspaces/models/user.py workspaces/models/valid.py workspaces/models/volume_storage.py diff --git a/applications/workspaces/server/test/services/test_dandi_upload.py b/applications/workspaces/server/test/services/test_dandi_upload.py new file mode 100644 index 000000000..d42fab1f1 --- /dev/null +++ b/applications/workspaces/server/test/services/test_dandi_upload.py @@ -0,0 +1,123 @@ +"""Tests for the EMBER-DANDI write-side adapter (IDP-43). + +Focused on the branching that actually bit us during live development against the real archive: +the two 409 paths (content already uploaded / asset already registered) and error surfacing. +The happy paths are thin wrappers over `requests` and are covered incidentally. + +Mirrors test_dandi_adapter.py's setup: `responses` for HTTP mocking, CH_VALUES_PATH so +cloudharness config resolves. +""" +import os + +import pytest +import responses + +from workspaces.service.osbrepository.adapters import dandi_upload + +HERE = os.path.dirname(os.path.realpath(__file__)) +os.environ["CH_VALUES_PATH"] = os.path.join(os.path.dirname(HERE), "values.yaml") + +API = dandi_upload.DANDI_API_BASE +DANDISET = dandi_upload.DANDI_DANDISET_ID + +ETAG = "0" * 32 + "-1" +BLOB_ID = "11111111-1111-1111-1111-111111111111" +ASSET_ID = "22222222-2222-2222-2222-222222222222" + + +@pytest.fixture(autouse=True) +def _no_real_secret(monkeypatch): + """The admin key is a mounted k8s secret that does not exist in a test environment.""" + monkeypatch.setattr(dandi_upload, "get_dandi_api_key", lambda: "test-token") + + +@responses.activate +def test_initialize_upload_returns_parts(): + responses.add( + responses.POST, + f"{API}/uploads/initialize/", + json={"upload_id": "up-1", "parts": [{"part_number": 1, "size": 10, "upload_url": "https://s3/part1"}]}, + status=200, + ) + + result = dandi_upload.initialize_upload(size=10, dandi_etag=ETAG) + + assert result["upload_id"] == "up-1" + assert len(result["parts"]) == 1 + assert result["parts"][0]["upload_url"] == "https://s3/part1" + + +@responses.activate +def test_initialize_upload_409_resolves_existing_blob(): + """A 409 means the archive already holds this exact content — not an error. We resolve the + existing blob so the caller can skip the S3 upload entirely and just register a new asset.""" + responses.add(responses.POST, f"{API}/uploads/initialize/", json={}, status=409) + responses.add(responses.POST, f"{API}/blobs/digest/", json={"blob_id": BLOB_ID}, status=200) + + result = dandi_upload.initialize_upload(size=10, dandi_etag=ETAG) + + assert result["blob_id"] == BLOB_ID + assert result["upload_id"] is None + assert result["parts"] == [] + + +@responses.activate +def test_initialize_upload_409_without_matching_blob_raises(): + """409 but the digest lookup finds nothing — genuinely inconsistent, must not pass silently.""" + responses.add(responses.POST, f"{API}/uploads/initialize/", json={}, status=409) + responses.add(responses.POST, f"{API}/blobs/digest/", json={}, status=404) + + with pytest.raises(RuntimeError, match="no blob matches digest"): + dandi_upload.initialize_upload(size=10, dandi_etag=ETAG) + + +@responses.activate +def test_register_asset_409_falls_back_to_existing(): + """Registering the same path twice (a retry after a later step failed) must be idempotent + rather than an error — otherwise a single transient failure poisons that path forever.""" + path = "task-x/sub-y/f.zip" + assets_url = f"{API}/dandisets/{DANDISET}/versions/draft/assets/" + + responses.add(responses.POST, assets_url, json={}, status=409) + responses.add( + responses.GET, + assets_url, + json={"results": [{"asset_id": ASSET_ID, "path": path}]}, + status=200, + ) + + asset = dandi_upload.register_asset(path=path, blob_id=BLOB_ID) + + assert asset["asset_id"] == ASSET_ID + assert asset["path"] == path + + +@responses.activate +def test_get_asset_by_path_requires_an_exact_match(): + """The ?path= filter is not assumed to return exact matches only, so this re-checks each + result — a near-miss must resolve to None rather than being mistaken for ours, since the + caller uses it to decide an upload was already registered.""" + path = "task-x/sub-y/f.zip" + responses.add( + responses.GET, + f"{API}/dandisets/{DANDISET}/versions/draft/assets/", + json={"results": [{"asset_id": ASSET_ID, "path": path + ".extra"}]}, + status=200, + ) + + assert dandi_upload.get_asset_by_path(path) is None + + +@responses.activate +def test_errors_surface_the_response_body(): + """raise_for_status() alone discards DANDI's explanation, which turns every 4xx into a + guessing game. The body is the whole reason these failures are debuggable.""" + responses.add( + responses.POST, + f"{API}/uploads/initialize/", + json={"non_field_errors": ["Digest improperly formatted"]}, + status=400, + ) + + with pytest.raises(RuntimeError, match="Digest improperly formatted"): + dandi_upload.initialize_upload(size=10, dandi_etag=ETAG) diff --git a/applications/workspaces/server/workspaces/controllers/dandi_upload_controller.py b/applications/workspaces/server/workspaces/controllers/dandi_upload_controller.py new file mode 100644 index 000000000..dd0f7ae8b --- /dev/null +++ b/applications/workspaces/server/workspaces/controllers/dandi_upload_controller.py @@ -0,0 +1,206 @@ +"""Brokers browser uploads into EMBER-DANDI, then attaches the result to a workspace and runs +the selected protocol's analysis script in it. + +These endpoints exist so the file's bytes can go browser -> S3 directly via presigned part +URLs, never through this server: a 500 MB recording costs this process a few hundred bytes of +JSON. The DANDI admin key stays server-side and never reaches the browser, which is why the +upload has to be brokered here rather than called from the frontend. +""" +import re + +import connexion +from cloudharness import log as logger + +from workspaces.models.upload_finalize_request import UploadFinalizeRequest +from workspaces.models.upload_finalize_response import UploadFinalizeResponse +from workspaces.models.upload_init_request import UploadInitRequest +from workspaces.models.upload_init_response import UploadInitResponse +from workspaces.models.upload_part import UploadPart +from workspaces.service import jupyter_kernel_client +from workspaces.service.auth import keycloak_user_id +from workspaces.service.crud_service import NotAllowed, NotAuthorized, NotFoundException, WorkspaceService +from workspaces.service.osbrepository.adapters import dandi_upload + + +def _bearer_token() -> str: + """The caller's raw access token. Needed verbatim (not just the decoded claims) because + JupyterHub's `/hub/chkclogin` reads it from a cookie — see jupyter_kernel_client.""" + header = connexion.request.headers.get("Authorization", "") + return header.split(" ", 1)[1] if header.lower().startswith("bearer ") else header + + +def _current_user_id() -> str: + """The keycloak user id of the caller — also the JupyterHub username, since the hub's + `chauthenticator` does `user_from_username(user_data['sub'])`.""" + user_id = keycloak_user_id() + if not user_id: + raise NotAuthorized("Could not resolve the current user from the request token") + return user_id + + +def _path_safe(value: str) -> str: + """Makes an identity usable as a DANDI asset path segment. + + DANDI rejects `@` with `400 {"non_field_errors":["Path improperly formatted"]}`, while `.`, + `-` and `_` are all accepted. Matters whenever the identity falls back to an email-shaped + username. Also the only thing standing between a client-supplied `filename` and a `../` + path-traversal segment — every `/` it might contain is replaced too. + """ + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", value or "") + return cleaned.strip("-._") or "unknown" + + +def _build_asset_path(task_id: str, sub: str, filename: str) -> str: + """The only place allowed to construct a DANDI asset path. Every segment goes through + `_path_safe`, so an unset `task_id` or a `filename` containing `../` can't produce a path + outside `task-*/sub-/`.""" + return f"task-{_path_safe(task_id)}/sub-{_path_safe(sub)}/{_path_safe(filename)}" + + +def _assert_path_owned_by_caller(path: str, sub: str) -> None: + """`finalize`'s request carries no `task_id`/`filename` to re-derive the path from — it only + echoes back the `path` `init` returned. So instead of trusting that echo outright, check it + still sits under the caller's own `sub-` segment before anything is registered under the + one shared DANDI admin key. Also rejects any extra `/` segments a doctored `path` might add. + """ + segments = path.split("/") + if len(segments) != 3 or segments[1] != f"sub-{_path_safe(sub)}": + raise NotAllowed(f"path does not belong to the current user: {path!r}") + + +def dandi_upload_init(body): + """POST /dandi/upload/init — reserve an upload in DANDI, hand back presigned S3 part URLs.""" + try: + return _dandi_upload_init(body) + except NotAuthorized as exc: + return str(exc) or "Not authorized", 401 + except NotAllowed as exc: + return str(exc) or "Not allowed", 405 + except NotFoundException as exc: + return str(exc) or "Not found", 404 + + +def _dandi_upload_init(body): + upload_init_request = UploadInitRequest.from_dict(body) + + # Path is derived server-side from the caller's identity, never taken from the request body. + # With one shared admin key, this is what stops one collaborator overwriting another's data. + path = _build_asset_path(upload_init_request.task_id, _current_user_id(), upload_init_request.filename) + + init_response = dandi_upload.initialize_upload( + size=upload_init_request.size, + dandi_etag=upload_init_request.dandi_etag, + ) + + # On the deduplicated path init_response carries no upload_id and no parts, just the + # existing blob_id — the client skips the S3 upload and goes straight to finalize. + return UploadInitResponse( + upload_id=init_response.get("upload_id"), + path=path, + blob_id=init_response.get("blob_id"), + parts=[ + UploadPart(part_number=p["part_number"], url=p["upload_url"]) + for p in init_response["parts"] + ], + ) + + +def dandi_upload_finalize(body): + """POST /dandi/upload/finalize — complete the DANDI upload, create/attach the workspace, + then run the protocol script in it. + + Synchronous end to end: this does NOT return until the script has finished, which makes the + request minutes-long in the worst case — the ingress/gunicorn timeouts in deploy/values.yaml + are set accordingly. See jupyter_kernel_client's docstring. + """ + try: + return _dandi_upload_finalize(body) + except NotAuthorized as exc: + return str(exc) or "Not authorized", 401 + except NotAllowed as exc: + return str(exc) or "Not allowed", 405 + except NotFoundException as exc: + return str(exc) or "Not found", 404 + + +def _dandi_upload_finalize(body): + upload_finalize_request = UploadFinalizeRequest.from_dict(body) + sub = _current_user_id() + _assert_path_owned_by_caller(upload_finalize_request.path, sub) + + if upload_finalize_request.blob_id: + # Deduplicated: the content was already in the archive, so no upload happened and + # there is nothing to complete or validate — just attach a new asset to the blob. + blob_id = upload_finalize_request.blob_id + else: + if not upload_finalize_request.parts: + return "parts is required when blob_id is not set", 400 + parts = [ + {"part_number": p.part_number, "size": p.size, "etag": p.etag} + for p in upload_finalize_request.parts + ] + dandi_upload.complete_upload(upload_finalize_request.upload_id, parts) + # validate_upload is what turns the completed multipart upload into a real AssetBlob in + # DANDI — required even though we no longer read the size off it. + blob_id = dandi_upload.validate_upload(upload_finalize_request.upload_id)["blob_id"] + + asset = dandi_upload.register_asset(upload_finalize_request.path, blob_id) + + # ── Workspace: in-process, no HTTP hop ──────────────────────────────────────────────── + # WorkspaceService.post() fills user_id from the current token itself and provisions the + # PVC synchronously (create_volume in the same call), so by the time this returns the + # workspace volume exists and can be written to. + # + # The data is deliberately NOT copied onto that volume: the protocol script fetches it + # straight from DANDI at run time, so a PVC copy would move the same bytes twice and leave a + # second permanent copy of every dataset behind. The workspace holds only the script and its + # outputs. Re-adding the copy means re-adding both the OSBRepository row and copy_origins() + # together — they are a pair, not independent steps. + workspace_id = upload_finalize_request.workspace_id + if workspace_id is None: + # `description` is REQUIRED by the Workspace schema alongside `name` — omitting it fails + # validation in WorkspaceService.post with MalformedModelDictionaryError, not with a + # helpful message about the missing field. + workspace_name = upload_finalize_request.workspace_name or upload_finalize_request.path + workspace = WorkspaceService().post({ + "name": workspace_name, + "description": workspace_name, + }) + workspace_id = workspace.id + else: + # An existing workspace_id is client-supplied, so confirm it's both real and the + # caller's own before spawning a pod and mounting its PVC — otherwise any id gets a + # spawn/execute inside (and a read mount of) whatever workspace happens to own it. + workspace_service = WorkspaceService() + existing = workspace_service.repository.get(workspace_id) + if existing is None: + raise NotFoundException(f"Workspace with id {workspace_id} not found.") + if not workspace_service.is_authorized(existing): + raise NotAuthorized() + + # ── Run the protocol script ─────────────────────────────────────────────────────────── + # Best-effort: the data is already in DANDI and the workspace already exists by this point, + # so a failure here should not report a successful upload as failed — it surfaces as + # script_output explaining what went wrong instead. + script_output = None + if upload_finalize_request.script_url: + try: + script_output = jupyter_kernel_client.run_script_in_workspace( + token=_bearer_token(), + user_id=sub, + workspace_id=workspace_id, + script_url=upload_finalize_request.script_url, + script_name=upload_finalize_request.script_name or "analysis.py", + asset_path=asset["path"], + dandiset_id=dandi_upload.DANDI_DANDISET_ID, + ) + except Exception as exc: # noqa: BLE001 — see comment above + logger.error("Script run failed (the upload itself succeeded)", exc_info=True) + script_output = f"Uploaded to DANDI, but the script did not run: {exc}" + + return UploadFinalizeResponse( + asset_path=asset["path"], + dandiset_url=dandi_upload.dandiset_url(), + workspace_id=workspace_id, + script_output=script_output, + ) diff --git a/applications/workspaces/server/workspaces/models/__init__.py b/applications/workspaces/server/workspaces/models/__init__.py index 9a1bd7f5d..66e2c7578 100644 --- a/applications/workspaces/server/workspaces/models/__init__.py +++ b/applications/workspaces/server/workspaces/models/__init__.py @@ -46,3 +46,10 @@ from workspaces.models.workspace_resource_entity_all_of import WorkspaceResourceEntityAllOf from workspaces.models.repository_info import RepositoryInfo from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource +# IDP-43 DANDI upload (ported from the standalone idp-arc backend, 2026-09-14) +from workspaces.models.upload_part import UploadPart +from workspaces.models.uploaded_part import UploadedPart +from workspaces.models.upload_init_request import UploadInitRequest +from workspaces.models.upload_init_response import UploadInitResponse +from workspaces.models.upload_finalize_request import UploadFinalizeRequest +from workspaces.models.upload_finalize_response import UploadFinalizeResponse diff --git a/applications/workspaces/server/workspaces/models/upload_finalize_request.py b/applications/workspaces/server/workspaces/models/upload_finalize_request.py new file mode 100644 index 000000000..e587731f9 --- /dev/null +++ b/applications/workspaces/server/workspaces/models/upload_finalize_request.py @@ -0,0 +1,264 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces.models.uploaded_part import UploadedPart +from workspaces import util + +from workspaces.models.uploaded_part import UploadedPart # noqa: E501 + +class UploadFinalizeRequest(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, upload_id=None, path=None, parts=None, blob_id=None, workspace_id=None, workspace_name=None, script_url=None, script_name=None): # noqa: E501 + """UploadFinalizeRequest - a model defined in OpenAPI + + :param upload_id: The upload_id of this UploadFinalizeRequest. # noqa: E501 + :type upload_id: str + :param path: The path of this UploadFinalizeRequest. # noqa: E501 + :type path: str + :param parts: The parts of this UploadFinalizeRequest. # noqa: E501 + :type parts: List[UploadedPart] + :param blob_id: The blob_id of this UploadFinalizeRequest. # noqa: E501 + :type blob_id: str + :param workspace_id: The workspace_id of this UploadFinalizeRequest. # noqa: E501 + :type workspace_id: int + :param workspace_name: The workspace_name of this UploadFinalizeRequest. # noqa: E501 + :type workspace_name: str + :param script_url: The script_url of this UploadFinalizeRequest. # noqa: E501 + :type script_url: str + :param script_name: The script_name of this UploadFinalizeRequest. # noqa: E501 + :type script_name: str + """ + self.openapi_types = { + 'upload_id': str, + 'path': str, + 'parts': List[UploadedPart], + 'blob_id': str, + 'workspace_id': int, + 'workspace_name': str, + 'script_url': str, + 'script_name': str + } + + self.attribute_map = { + 'upload_id': 'upload_id', + 'path': 'path', + 'parts': 'parts', + 'blob_id': 'blob_id', + 'workspace_id': 'workspace_id', + 'workspace_name': 'workspace_name', + 'script_url': 'script_url', + 'script_name': 'script_name' + } + + self._upload_id = upload_id + self._path = path + self._parts = parts + self._blob_id = blob_id + self._workspace_id = workspace_id + self._workspace_name = workspace_name + self._script_url = script_url + self._script_name = script_name + + @classmethod + def from_dict(cls, dikt) -> 'UploadFinalizeRequest': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The UploadFinalizeRequest of this UploadFinalizeRequest. # noqa: E501 + :rtype: UploadFinalizeRequest + """ + return util.deserialize_model(dikt, cls) + + @property + def upload_id(self): + """Gets the upload_id of this UploadFinalizeRequest. + + Omitted on the deduplicated path, where no upload took place. # noqa: E501 + + :return: The upload_id of this UploadFinalizeRequest. + :rtype: str + """ + return self._upload_id + + @upload_id.setter + def upload_id(self, upload_id): + """Sets the upload_id of this UploadFinalizeRequest. + + Omitted on the deduplicated path, where no upload took place. # noqa: E501 + + :param upload_id: The upload_id of this UploadFinalizeRequest. + :type upload_id: str + """ + + self._upload_id = upload_id + + @property + def path(self): + """Gets the path of this UploadFinalizeRequest. + + + :return: The path of this UploadFinalizeRequest. + :rtype: str + """ + return self._path + + @path.setter + def path(self, path): + """Sets the path of this UploadFinalizeRequest. + + + :param path: The path of this UploadFinalizeRequest. + :type path: str + """ + if path is None: + raise ValueError("Invalid value for `path`, must not be `None`") # noqa: E501 + + self._path = path + + @property + def parts(self): + """Gets the parts of this UploadFinalizeRequest. + + Omitted on the deduplicated path. # noqa: E501 + + :return: The parts of this UploadFinalizeRequest. + :rtype: List[UploadedPart] + """ + return self._parts + + @parts.setter + def parts(self, parts): + """Sets the parts of this UploadFinalizeRequest. + + Omitted on the deduplicated path. # noqa: E501 + + :param parts: The parts of this UploadFinalizeRequest. + :type parts: List[UploadedPart] + """ + + self._parts = parts + + @property + def blob_id(self): + """Gets the blob_id of this UploadFinalizeRequest. + + Echoed back from init's deduplicated response; when present, complete/validate are skipped. # noqa: E501 + + :return: The blob_id of this UploadFinalizeRequest. + :rtype: str + """ + return self._blob_id + + @blob_id.setter + def blob_id(self, blob_id): + """Sets the blob_id of this UploadFinalizeRequest. + + Echoed back from init's deduplicated response; when present, complete/validate are skipped. # noqa: E501 + + :param blob_id: The blob_id of this UploadFinalizeRequest. + :type blob_id: str + """ + + self._blob_id = blob_id + + @property + def workspace_id(self): + """Gets the workspace_id of this UploadFinalizeRequest. + + Existing workspace to attach to. Omit to create a new one. # noqa: E501 + + :return: The workspace_id of this UploadFinalizeRequest. + :rtype: int + """ + return self._workspace_id + + @workspace_id.setter + def workspace_id(self, workspace_id): + """Sets the workspace_id of this UploadFinalizeRequest. + + Existing workspace to attach to. Omit to create a new one. # noqa: E501 + + :param workspace_id: The workspace_id of this UploadFinalizeRequest. + :type workspace_id: int + """ + + self._workspace_id = workspace_id + + @property + def workspace_name(self): + """Gets the workspace_name of this UploadFinalizeRequest. + + Name for a new workspace, used only when workspace_id is omitted. # noqa: E501 + + :return: The workspace_name of this UploadFinalizeRequest. + :rtype: str + """ + return self._workspace_name + + @workspace_name.setter + def workspace_name(self, workspace_name): + """Sets the workspace_name of this UploadFinalizeRequest. + + Name for a new workspace, used only when workspace_id is omitted. # noqa: E501 + + :param workspace_name: The workspace_name of this UploadFinalizeRequest. + :type workspace_name: str + """ + + self._workspace_name = workspace_name + + @property + def script_url(self): + """Gets the script_url of this UploadFinalizeRequest. + + Publicly-reachable URL of the protocol's analysis script. Fetched and run server-side inside the workspace; omit to skip running anything. # noqa: E501 + + :return: The script_url of this UploadFinalizeRequest. + :rtype: str + """ + return self._script_url + + @script_url.setter + def script_url(self, script_url): + """Sets the script_url of this UploadFinalizeRequest. + + Publicly-reachable URL of the protocol's analysis script. Fetched and run server-side inside the workspace; omit to skip running anything. # noqa: E501 + + :param script_url: The script_url of this UploadFinalizeRequest. + :type script_url: str + """ + + self._script_url = script_url + + @property + def script_name(self): + """Gets the script_name of this UploadFinalizeRequest. + + Filename the script lands under in the workspace. # noqa: E501 + + :return: The script_name of this UploadFinalizeRequest. + :rtype: str + """ + return self._script_name + + @script_name.setter + def script_name(self, script_name): + """Sets the script_name of this UploadFinalizeRequest. + + Filename the script lands under in the workspace. # noqa: E501 + + :param script_name: The script_name of this UploadFinalizeRequest. + :type script_name: str + """ + + self._script_name = script_name diff --git a/applications/workspaces/server/workspaces/models/upload_finalize_response.py b/applications/workspaces/server/workspaces/models/upload_finalize_response.py new file mode 100644 index 000000000..66de59cc8 --- /dev/null +++ b/applications/workspaces/server/workspaces/models/upload_finalize_response.py @@ -0,0 +1,150 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces import util + + +class UploadFinalizeResponse(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, asset_path=None, dandiset_url=None, workspace_id=None, script_output=None): # noqa: E501 + """UploadFinalizeResponse - a model defined in OpenAPI + + :param asset_path: The asset_path of this UploadFinalizeResponse. # noqa: E501 + :type asset_path: str + :param dandiset_url: The dandiset_url of this UploadFinalizeResponse. # noqa: E501 + :type dandiset_url: str + :param workspace_id: The workspace_id of this UploadFinalizeResponse. # noqa: E501 + :type workspace_id: int + :param script_output: The script_output of this UploadFinalizeResponse. # noqa: E501 + :type script_output: str + """ + self.openapi_types = { + 'asset_path': str, + 'dandiset_url': str, + 'workspace_id': int, + 'script_output': str + } + + self.attribute_map = { + 'asset_path': 'asset_path', + 'dandiset_url': 'dandiset_url', + 'workspace_id': 'workspace_id', + 'script_output': 'script_output' + } + + self._asset_path = asset_path + self._dandiset_url = dandiset_url + self._workspace_id = workspace_id + self._script_output = script_output + + @classmethod + def from_dict(cls, dikt) -> 'UploadFinalizeResponse': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The UploadFinalizeResponse of this UploadFinalizeResponse. # noqa: E501 + :rtype: UploadFinalizeResponse + """ + return util.deserialize_model(dikt, cls) + + @property + def asset_path(self): + """Gets the asset_path of this UploadFinalizeResponse. + + Path of the registered DANDI asset. # noqa: E501 + + :return: The asset_path of this UploadFinalizeResponse. + :rtype: str + """ + return self._asset_path + + @asset_path.setter + def asset_path(self, asset_path): + """Sets the asset_path of this UploadFinalizeResponse. + + Path of the registered DANDI asset. # noqa: E501 + + :param asset_path: The asset_path of this UploadFinalizeResponse. + :type asset_path: str + """ + if asset_path is None: + raise ValueError("Invalid value for `asset_path`, must not be `None`") # noqa: E501 + + self._asset_path = asset_path + + @property + def dandiset_url(self): + """Gets the dandiset_url of this UploadFinalizeResponse. + + + :return: The dandiset_url of this UploadFinalizeResponse. + :rtype: str + """ + return self._dandiset_url + + @dandiset_url.setter + def dandiset_url(self, dandiset_url): + """Sets the dandiset_url of this UploadFinalizeResponse. + + + :param dandiset_url: The dandiset_url of this UploadFinalizeResponse. + :type dandiset_url: str + """ + + self._dandiset_url = dandiset_url + + @property + def workspace_id(self): + """Gets the workspace_id of this UploadFinalizeResponse. + + + :return: The workspace_id of this UploadFinalizeResponse. + :rtype: int + """ + return self._workspace_id + + @workspace_id.setter + def workspace_id(self, workspace_id): + """Sets the workspace_id of this UploadFinalizeResponse. + + + :param workspace_id: The workspace_id of this UploadFinalizeResponse. + :type workspace_id: int + """ + if workspace_id is None: + raise ValueError("Invalid value for `workspace_id`, must not be `None`") # noqa: E501 + + self._workspace_id = workspace_id + + @property + def script_output(self): + """Gets the script_output of this UploadFinalizeResponse. + + Everything the protocol script printed, or an explanation of why it did not run. Absent when no script_url was supplied. # noqa: E501 + + :return: The script_output of this UploadFinalizeResponse. + :rtype: str + """ + return self._script_output + + @script_output.setter + def script_output(self, script_output): + """Sets the script_output of this UploadFinalizeResponse. + + Everything the protocol script printed, or an explanation of why it did not run. Absent when no script_url was supplied. # noqa: E501 + + :param script_output: The script_output of this UploadFinalizeResponse. + :type script_output: str + """ + + self._script_output = script_output diff --git a/applications/workspaces/server/workspaces/models/upload_init_request.py b/applications/workspaces/server/workspaces/models/upload_init_request.py new file mode 100644 index 000000000..329881ed0 --- /dev/null +++ b/applications/workspaces/server/workspaces/models/upload_init_request.py @@ -0,0 +1,152 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces import util + + +class UploadInitRequest(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, task_id=None, filename=None, size=None, dandi_etag=None): # noqa: E501 + """UploadInitRequest - a model defined in OpenAPI + + :param task_id: The task_id of this UploadInitRequest. # noqa: E501 + :type task_id: str + :param filename: The filename of this UploadInitRequest. # noqa: E501 + :type filename: str + :param size: The size of this UploadInitRequest. # noqa: E501 + :type size: int + :param dandi_etag: The dandi_etag of this UploadInitRequest. # noqa: E501 + :type dandi_etag: str + """ + self.openapi_types = { + 'task_id': str, + 'filename': str, + 'size': int, + 'dandi_etag': str + } + + self.attribute_map = { + 'task_id': 'task_id', + 'filename': 'filename', + 'size': 'size', + 'dandi_etag': 'dandi_etag' + } + + self._task_id = task_id + self._filename = filename + self._size = size + self._dandi_etag = dandi_etag + + @classmethod + def from_dict(cls, dikt) -> 'UploadInitRequest': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The UploadInitRequest of this UploadInitRequest. # noqa: E501 + :rtype: UploadInitRequest + """ + return util.deserialize_model(dikt, cls) + + @property + def task_id(self): + """Gets the task_id of this UploadInitRequest. + + Slug of the selected protocol; becomes the first segment of the asset path. # noqa: E501 + + :return: The task_id of this UploadInitRequest. + :rtype: str + """ + return self._task_id + + @task_id.setter + def task_id(self, task_id): + """Sets the task_id of this UploadInitRequest. + + Slug of the selected protocol; becomes the first segment of the asset path. # noqa: E501 + + :param task_id: The task_id of this UploadInitRequest. + :type task_id: str + """ + + self._task_id = task_id + + @property + def filename(self): + """Gets the filename of this UploadInitRequest. + + + :return: The filename of this UploadInitRequest. + :rtype: str + """ + return self._filename + + @filename.setter + def filename(self, filename): + """Sets the filename of this UploadInitRequest. + + + :param filename: The filename of this UploadInitRequest. + :type filename: str + """ + if filename is None: + raise ValueError("Invalid value for `filename`, must not be `None`") # noqa: E501 + + self._filename = filename + + @property + def size(self): + """Gets the size of this UploadInitRequest. + + + :return: The size of this UploadInitRequest. + :rtype: int + """ + return self._size + + @size.setter + def size(self, size): + """Sets the size of this UploadInitRequest. + + + :param size: The size of this UploadInitRequest. + :type size: int + """ + if size is None: + raise ValueError("Invalid value for `size`, must not be `None`") # noqa: E501 + + self._size = size + + @property + def dandi_etag(self): + """Gets the dandi_etag of this UploadInitRequest. + + S3-multipart digest in DANDI's format (32 hex chars, a dash, the part count), computed in the browser since the bytes never reach this server. # noqa: E501 + + :return: The dandi_etag of this UploadInitRequest. + :rtype: str + """ + return self._dandi_etag + + @dandi_etag.setter + def dandi_etag(self, dandi_etag): + """Sets the dandi_etag of this UploadInitRequest. + + S3-multipart digest in DANDI's format (32 hex chars, a dash, the part count), computed in the browser since the bytes never reach this server. # noqa: E501 + + :param dandi_etag: The dandi_etag of this UploadInitRequest. + :type dandi_etag: str + """ + if dandi_etag is None: + raise ValueError("Invalid value for `dandi_etag`, must not be `None`") # noqa: E501 + + self._dandi_etag = dandi_etag diff --git a/applications/workspaces/server/workspaces/models/upload_init_response.py b/applications/workspaces/server/workspaces/models/upload_init_response.py new file mode 100644 index 000000000..07dda5185 --- /dev/null +++ b/applications/workspaces/server/workspaces/models/upload_init_response.py @@ -0,0 +1,150 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces.models.upload_part import UploadPart +from workspaces import util + +from workspaces.models.upload_part import UploadPart # noqa: E501 + +class UploadInitResponse(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, upload_id=None, path=None, parts=None, blob_id=None): # noqa: E501 + """UploadInitResponse - a model defined in OpenAPI + + :param upload_id: The upload_id of this UploadInitResponse. # noqa: E501 + :type upload_id: str + :param path: The path of this UploadInitResponse. # noqa: E501 + :type path: str + :param parts: The parts of this UploadInitResponse. # noqa: E501 + :type parts: List[UploadPart] + :param blob_id: The blob_id of this UploadInitResponse. # noqa: E501 + :type blob_id: str + """ + self.openapi_types = { + 'upload_id': str, + 'path': str, + 'parts': List[UploadPart], + 'blob_id': str + } + + self.attribute_map = { + 'upload_id': 'upload_id', + 'path': 'path', + 'parts': 'parts', + 'blob_id': 'blob_id' + } + + self._upload_id = upload_id + self._path = path + self._parts = parts + self._blob_id = blob_id + + @classmethod + def from_dict(cls, dikt) -> 'UploadInitResponse': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The UploadInitResponse of this UploadInitResponse. # noqa: E501 + :rtype: UploadInitResponse + """ + return util.deserialize_model(dikt, cls) + + @property + def upload_id(self): + """Gets the upload_id of this UploadInitResponse. + + Absent when the content was already in the archive. # noqa: E501 + + :return: The upload_id of this UploadInitResponse. + :rtype: str + """ + return self._upload_id + + @upload_id.setter + def upload_id(self, upload_id): + """Sets the upload_id of this UploadInitResponse. + + Absent when the content was already in the archive. # noqa: E501 + + :param upload_id: The upload_id of this UploadInitResponse. + :type upload_id: str + """ + + self._upload_id = upload_id + + @property + def path(self): + """Gets the path of this UploadInitResponse. + + Asset path derived server-side from the caller's identity. # noqa: E501 + + :return: The path of this UploadInitResponse. + :rtype: str + """ + return self._path + + @path.setter + def path(self, path): + """Sets the path of this UploadInitResponse. + + Asset path derived server-side from the caller's identity. # noqa: E501 + + :param path: The path of this UploadInitResponse. + :type path: str + """ + + self._path = path + + @property + def parts(self): + """Gets the parts of this UploadInitResponse. + + + :return: The parts of this UploadInitResponse. + :rtype: List[UploadPart] + """ + return self._parts + + @parts.setter + def parts(self, parts): + """Sets the parts of this UploadInitResponse. + + + :param parts: The parts of this UploadInitResponse. + :type parts: List[UploadPart] + """ + + self._parts = parts + + @property + def blob_id(self): + """Gets the blob_id of this UploadInitResponse. + + Set only on the deduplicated path. # noqa: E501 + + :return: The blob_id of this UploadInitResponse. + :rtype: str + """ + return self._blob_id + + @blob_id.setter + def blob_id(self, blob_id): + """Sets the blob_id of this UploadInitResponse. + + Set only on the deduplicated path. # noqa: E501 + + :param blob_id: The blob_id of this UploadInitResponse. + :type blob_id: str + """ + + self._blob_id = blob_id diff --git a/applications/workspaces/server/workspaces/models/upload_part.py b/applications/workspaces/server/workspaces/models/upload_part.py new file mode 100644 index 000000000..2cb5e6291 --- /dev/null +++ b/applications/workspaces/server/workspaces/models/upload_part.py @@ -0,0 +1,92 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces import util + + +class UploadPart(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, part_number=None, url=None): # noqa: E501 + """UploadPart - a model defined in OpenAPI + + :param part_number: The part_number of this UploadPart. # noqa: E501 + :type part_number: int + :param url: The url of this UploadPart. # noqa: E501 + :type url: str + """ + self.openapi_types = { + 'part_number': int, + 'url': str + } + + self.attribute_map = { + 'part_number': 'part_number', + 'url': 'url' + } + + self._part_number = part_number + self._url = url + + @classmethod + def from_dict(cls, dikt) -> 'UploadPart': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The UploadPart of this UploadPart. # noqa: E501 + :rtype: UploadPart + """ + return util.deserialize_model(dikt, cls) + + @property + def part_number(self): + """Gets the part_number of this UploadPart. + + + :return: The part_number of this UploadPart. + :rtype: int + """ + return self._part_number + + @part_number.setter + def part_number(self, part_number): + """Sets the part_number of this UploadPart. + + + :param part_number: The part_number of this UploadPart. + :type part_number: int + """ + + self._part_number = part_number + + @property + def url(self): + """Gets the url of this UploadPart. + + Presigned S3 URL the browser PUTs this part's bytes to directly. # noqa: E501 + + :return: The url of this UploadPart. + :rtype: str + """ + return self._url + + @url.setter + def url(self, url): + """Sets the url of this UploadPart. + + Presigned S3 URL the browser PUTs this part's bytes to directly. # noqa: E501 + + :param url: The url of this UploadPart. + :type url: str + """ + + self._url = url diff --git a/applications/workspaces/server/workspaces/models/uploaded_part.py b/applications/workspaces/server/workspaces/models/uploaded_part.py new file mode 100644 index 000000000..53f1eb5db --- /dev/null +++ b/applications/workspaces/server/workspaces/models/uploaded_part.py @@ -0,0 +1,118 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from workspaces.models.base_model_ import Model +from workspaces import util + + +class UploadedPart(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, part_number=None, size=None, etag=None): # noqa: E501 + """UploadedPart - a model defined in OpenAPI + + :param part_number: The part_number of this UploadedPart. # noqa: E501 + :type part_number: int + :param size: The size of this UploadedPart. # noqa: E501 + :type size: int + :param etag: The etag of this UploadedPart. # noqa: E501 + :type etag: str + """ + self.openapi_types = { + 'part_number': int, + 'size': int, + 'etag': str + } + + self.attribute_map = { + 'part_number': 'part_number', + 'size': 'size', + 'etag': 'etag' + } + + self._part_number = part_number + self._size = size + self._etag = etag + + @classmethod + def from_dict(cls, dikt) -> 'UploadedPart': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The UploadedPart of this UploadedPart. # noqa: E501 + :rtype: UploadedPart + """ + return util.deserialize_model(dikt, cls) + + @property + def part_number(self): + """Gets the part_number of this UploadedPart. + + + :return: The part_number of this UploadedPart. + :rtype: int + """ + return self._part_number + + @part_number.setter + def part_number(self, part_number): + """Sets the part_number of this UploadedPart. + + + :param part_number: The part_number of this UploadedPart. + :type part_number: int + """ + + self._part_number = part_number + + @property + def size(self): + """Gets the size of this UploadedPart. + + + :return: The size of this UploadedPart. + :rtype: int + """ + return self._size + + @size.setter + def size(self, size): + """Sets the size of this UploadedPart. + + + :param size: The size of this UploadedPart. + :type size: int + """ + + self._size = size + + @property + def etag(self): + """Gets the etag of this UploadedPart. + + The ETag S3 returned for this part's PUT. # noqa: E501 + + :return: The etag of this UploadedPart. + :rtype: str + """ + return self._etag + + @etag.setter + def etag(self, etag): + """Sets the etag of this UploadedPart. + + The ETag S3 returned for this part's PUT. # noqa: E501 + + :param etag: The etag of this UploadedPart. + :type etag: str + """ + + self._etag = etag diff --git a/applications/workspaces/server/workspaces/openapi/openapi.yaml b/applications/workspaces/server/workspaces/openapi/openapi.yaml index f61b03690..8e619efc7 100644 --- a/applications/workspaces/server/workspaces/openapi/openapi.yaml +++ b/applications/workspaces/server/workspaces/openapi/openapi.yaml @@ -1015,8 +1015,172 @@ paths: operationId: user_resource_counts summary: Per-user workspace and repository counts (admin only). x-openapi-router-controller: workspaces.controllers.admin_controller + /dandi/upload/init: + post: + requestBody: + description: The file to reserve an upload slot for in EMBER-DANDI. + content: + application/json: + schema: + $ref: '#/components/schemas/UploadInitRequest' + required: true + tags: + - dandi + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UploadInitResponse' + description: >- + Upload reserved. Returns presigned S3 part URLs the browser PUTs to + directly, or (when the content already exists in the archive) an empty + parts list plus the existing blob_id. + '401': + description: Missing or invalid token + security: + - + bearerAuth: [] + operationId: workspaces.controllers.dandi_upload_controller.dandi_upload_init + summary: >- + Reserve a DANDI upload. The asset path is derived server-side from the caller's + identity and is never taken from the request body. + /dandi/upload/finalize: + post: + requestBody: + description: The completed upload to register, attach to a workspace, and analyse. + content: + application/json: + schema: + $ref: '#/components/schemas/UploadFinalizeRequest' + required: true + tags: + - dandi + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UploadFinalizeResponse' + description: Asset registered, workspace attached, script run. + '401': + description: Missing or invalid token + security: + - + bearerAuth: [] + operationId: workspaces.controllers.dandi_upload_controller.dandi_upload_finalize + summary: >- + Complete the DANDI upload, create/attach the workspace, and run the protocol + script in it. Synchronous — does not return until the script finishes, which + can take minutes. components: schemas: + UploadPart: + type: object + properties: + part_number: + type: integer + url: + type: string + description: Presigned S3 URL the browser PUTs this part's bytes to directly. + UploadedPart: + type: object + properties: + part_number: + type: integer + size: + type: integer + etag: + type: string + description: The ETag S3 returned for this part's PUT. + UploadInitRequest: + type: object + required: + - filename + - size + - dandi_etag + properties: + task_id: + type: string + description: >- + Slug of the selected protocol; becomes the first segment of the asset path. + filename: + type: string + size: + type: integer + dandi_etag: + type: string + description: >- + S3-multipart digest in DANDI's format (32 hex chars, a dash, the part + count), computed in the browser since the bytes never reach this server. + UploadInitResponse: + type: object + properties: + upload_id: + type: string + description: Absent when the content was already in the archive. + path: + type: string + description: Asset path derived server-side from the caller's identity. + parts: + type: array + items: + $ref: '#/components/schemas/UploadPart' + blob_id: + type: string + description: Set only on the deduplicated path. + UploadFinalizeRequest: + type: object + required: + - path + properties: + upload_id: + type: string + description: Omitted on the deduplicated path, where no upload took place. + path: + type: string + parts: + type: array + description: Omitted on the deduplicated path. + items: + $ref: '#/components/schemas/UploadedPart' + blob_id: + type: string + description: >- + Echoed back from init's deduplicated response; when present, + complete/validate are skipped. + workspace_id: + type: integer + description: Existing workspace to attach to. Omit to create a new one. + workspace_name: + type: string + description: Name for a new workspace, used only when workspace_id is omitted. + script_url: + type: string + description: >- + Publicly-reachable URL of the protocol's analysis script. Fetched and run + server-side inside the workspace; omit to skip running anything. + script_name: + type: string + description: Filename the script lands under in the workspace. + UploadFinalizeResponse: + type: object + required: + - asset_path + - workspace_id + properties: + asset_path: + type: string + description: Path of the registered DANDI asset. + dandiset_url: + type: string + workspace_id: + type: integer + script_output: + type: string + description: >- + Everything the protocol script printed, or an explanation of why it did + not run. Absent when no script_url was supplied. Valid: type: object properties: @@ -1582,3 +1746,8 @@ tags: - name: Client description: Client + - + name: dandi + description: >- + Brokered uploads into EMBER-DANDI (IDP-43). File bytes go browser -> S3 directly via + presigned part URLs; only metadata passes through this server. diff --git a/applications/workspaces/server/workspaces/service/jupyter_kernel_client.py b/applications/workspaces/server/workspaces/service/jupyter_kernel_client.py new file mode 100644 index 000000000..010f7952d --- /dev/null +++ b/applications/workspaces/server/workspaces/service/jupyter_kernel_client.py @@ -0,0 +1,495 @@ +"""Runs a script inside a workspace's JupyterLab server, from this backend, synchronously. + +Authenticates the way JupyterHub's own browser client does: `/hub/chkclogin` reads the access +token from a `kc-access`/`accessToken` COOKIE, never a header (see chauthenticator/auth.py), so +this still has to do the cookie dance — just with `requests.Session` instead of a browser. +`WorkspaceService.post()` already provisions the PVC synchronously, so nothing here needs to +poll for the volume, only for the JupyterLab pod to start. Kernel execution has no REST +equivalent — only lifecycle (`POST`/`DELETE /api/kernels`) is REST; running code is always the +`/api/kernels/{id}/channels` WebSocket, since a kernel streams stdout over time rather than +returning once. + +This runs inline inside the `dandi_upload_finalize` request and blocks until the script +finishes, rather than going through Argo. That makes the request minutes-long, so the +ingress/gunicorn timeouts (deploy/values.yaml) are raised to match and the budgets below are +kept under gunicorn's own TIMEOUT. Moving it to a workflow is the intended follow-up. +""" +import json +import logging +import os +import time +import uuid +from urllib.parse import urlsplit + +import requests +import websocket # websocket-client — already a workspaces/server dependency +from cloudharness.applications import get_configuration + +logger = logging.getLogger(__name__) + +# `script_url` is client-supplied and fetched server-side, from inside the cluster — an +# unrestricted GET here is SSRF, with the response handed straight back to the caller as +# script_output. Only hosts known to serve protocol scripts are allowed; anything else (in +# particular any cluster-internal hostname or IP) is refused before the request is made. +# Configurable per-deployment, since which hosts are legitimate depends on where the calling +# application publishes its protocol catalog. +ALLOWED_SCRIPT_HOSTS = frozenset( + h.strip() + for h in os.environ.get( + "WORKSPACES_ALLOWED_SCRIPT_HOSTS", "gist.githubusercontent.com,raw.githubusercontent.com" + ).split(",") + if h.strip() +) + +# The JupyterLab *application* whose named server we spawn. Its subdomain drives both values +# below, so keep them derived rather than hardcoded — they differ per environment. +# +# OSB ships two interchangeable JupyterLab apps and a deployment includes one or the other: +# `jupyterlab` (the full scientific image) and `jupyterlab-minimal`. They share a subdomain +# (`lab`, aliased to `notebooks`), so either answers the same URLs — but get_configuration() +# raises "Application X is not part of the current deployment" for whichever is absent. Try +# them in turn rather than assuming. +JUPYTER_APP_CANDIDATES = [ + n for n in [ + os.environ.get("WORKSPACES_JUPYTER_APP"), + "jupyterlab", + "jupyterlab-minimal", + ] if n +] + + +def _jupyter_app(): + """The deployed JupyterLab application's configuration, whichever variant is present.""" + errors = [] + for name in JUPYTER_APP_CANDIDATES: + try: + return get_configuration(name) + except Exception as exc: # noqa: BLE001 — cloudharness raises a bare Exception subclass + errors.append(f"{name}: {exc}") + raise ScriptRunError( + "No JupyterLab application found in this deployment; tried " + + ", ".join(JUPYTER_APP_CANDIDATES) + + ". Set WORKSPACES_JUPYTER_APP to the right app name. (" + + " | ".join(errors) + ")" + ) + + +def _jupyter_base() -> str: + """In-cluster URL of the JupyterLab app's proxy, from CloudHarness config. + + Deliberately the in-cluster SERVICE address (`get_service_address()`), NOT the public one + (`get_public_address()`). This code always runs server-side, inside the `workspaces` + pod, in the same cluster as JupyterHub's proxy — so it should talk to it directly, the way + every other in-cluster caller does, rather than round-tripping out through the ingress. The + public hostname can resolve back to the pod's own loopback via CoreDNS's default forwarding, + failing every call outright — going through the public ingress from inside the cluster is + the wrong direction regardless of environment. + + Overridable for the case where the hub is not part of this deployment at all and an + in-cluster Service address genuinely does not exist. + """ + override = os.environ.get("WORKSPACES_JUPYTER_BASE") + if override: + return override.rstrip("/") + return _jupyter_app().get_service_address().rstrip("/") + + +def _public_host() -> str: + """Hostname (no scheme) of the JupyterLab app's public address. + + cloud-harness's own `harness_jupyter.jupyterhub.change_pod_manifest` (and OSB's + `osb_jupyterhub.change_pod_manifest`) both pick the singleuser pod's image and resource + limits by string-splitting the incoming request's `Host` header on the deployment domain to + recover a subdomain, then matching that subdomain against each app's configured + `harness.subdomain`. That assumes every request to the hub arrives through the public + ingress, carrying the public subdomain in `Host`. + + We deliberately connect to the in-cluster Service address instead (see `_jupyter_base()`), + so the Host header the hub would otherwise see is the Service's own DNS name — which carries + no subdomain to split out, leaving the hub on its default app config and spawning the stock + sample image rather than the one this deployment wants. + + Fix: keep the working in-cluster connection, but send the real public hostname as an + explicit `Host` header override, so the hub's own app-detection sees what it expects while + the TCP connection still goes in-cluster. + """ + override = os.environ.get("WORKSPACES_JUPYTER_PUBLIC_HOST") + if override: + return override + from urllib.parse import urlparse + return urlparse(_jupyter_app().get_public_address()).hostname + + +def _server_suffix() -> str: + """The named-server suffix. OSB's own portal derives this as the first 4 characters of the + application's subdomain — `application.subdomain.slice(0, 4)` in + osb-portal/src/components/workspace/WorkspaceFrame.tsx. Mirrored here so both agree on the + server name; if they disagree we would spawn a second, parallel server per workspace. + + The subdomain differs between branches — `lab` on develop (workspace 764 -> `764lab`), + `notebooks` on master (-> `764note`) — hence deriving it rather than hardcoding either. + """ + override = os.environ.get("WORKSPACES_JUPYTER_SERVER_SUFFIX") + if override: + return override + return _jupyter_app().harness.subdomain[:4] + + +SPAWN_POLL_INTERVAL_S = 4 +# These two must sum to less than the gunicorn worker timeout (Dockerfile: TIMEOUT=360), since +# this whole sequence runs inside one synchronous request. Deliberately fitted inside OSB's +# existing envelope rather than raising a setting shared by every other workspaces endpoint — +# if real spawns turn out slower than 180s, raise TIMEOUT there and these together. +DEFAULT_SPAWN_TIMEOUT_S = 180 +DEFAULT_RUN_TIMEOUT_S = 150 + + +class ScriptRunError(RuntimeError): + pass + + +def run_script_in_workspace( + token: str, + user_id: str, + workspace_id: int, + script_url: str, + script_name: str, + asset_path: str | None = None, + dandiset_id: str | None = None, + spawn_timeout_s: int = DEFAULT_SPAWN_TIMEOUT_S, + run_timeout_s: int = DEFAULT_RUN_TIMEOUT_S, +) -> str: + """ + Spawns (or reuses) the workspace's JupyterLab server, writes the script into it, runs + it, and returns everything it printed. Blocks for the whole duration — see module docstring. + + `asset_path` and `dandiset_id` are passed into the kernel as `ASSET_PATH`/`DANDISET_ID` env + vars, which protocol scripts read in preference to any fallback of their own. Without them a + script analyzes whatever it defaults to rather than the data just uploaded, and has to carry + a dandiset ID in its own source to work at all. + """ + base = _jupyter_base() + server_name = f"{workspace_id}{_server_suffix()}" + session = requests.Session() + session.headers["Host"] = _public_host() + + script_text = _fetch_script(script_url) + _trigger_spawn(session, base, token, user_id, workspace_id, server_name) + _wait_until_ready(session, base, user_id, server_name, spawn_timeout_s) + _put_text_file(session, base, user_id, server_name, script_name, script_text) + return _run_script(session, base, user_id, server_name, script_name, run_timeout_s, asset_path, dandiset_id) + + +def _assert_script_url_allowed(script_url: str) -> None: + parsed = urlsplit(script_url) + if parsed.scheme != "https" or parsed.hostname not in ALLOWED_SCRIPT_HOSTS: + raise ScriptRunError(f"script_url host is not on the allowlist: {script_url!r}") + + +def _fetch_script(script_url: str) -> str: + _assert_script_url_allowed(script_url) + resp = requests.get(script_url, timeout=30) + if not resp.ok: + raise ScriptRunError(f"Could not fetch script from {script_url!r}: HTTP {resp.status_code}") + return resp.text + + +def _xsrf_for(session: requests.Session, user_id: str, server_name: str) -> str: + """The `_xsrf` cookie scoped to this singleuser server's own path. + + Once `_is_ready()`'s page-URL warm-up has run once, the cookie jar holds TWO cookies both + named `_xsrf` — Hub's own (path `/hub/`) and the singleuser server's own (path + `/user/{user_id}/{server_name}/`), which is the one its CSRF check actually validates + against. `session.cookies.get("_xsrf")` can't disambiguate same-named cookies on different + paths and raises `CookieConflictError`, a `RequestException` subclass — every caller's + `except requests.RequestException: return False` silently swallows it as "not ready yet" and + polls forever. Pick the cookie whose path matches this server explicitly. + """ + prefix = f"/user/{user_id}/{server_name}" + for cookie in session.cookies: + if cookie.name == "_xsrf" and (cookie.path or "").startswith(prefix): + return cookie.value + # Fall back to whatever's there (e.g. only Hub's own, before the per-server one exists yet). + matches = [c.value for c in session.cookies if c.name == "_xsrf"] + return matches[0] if matches else "" + + +def _is_ready(session: requests.Session, base: str, user_id: str, server_name: str) -> bool: + # The Hub-level session cookie from chkclogin is NOT enough to call the singleuser server's + # own API directly — JupyterHub fronts each spawned server with its own per-server OAuth + # client, and a request lacking that server-specific token gets a flat `403 Forbidden: No + # user identified` forever, never a redirect into the OAuth flow (that flow only triggers for + # a plain *page* request, not an API path). A plain GET of the server's own page URL redirects + # through /hub/api/oauth2/authorize -> consent -> callback and lands the per-server cookie, + # after which the real API works — so touch the page URL first (cheap, idempotent), then + # check readiness via the API. + page_url = f"{base}/user/{user_id}/{server_name}/" + try: + session.get(page_url, timeout=15) + except requests.RequestException as exc: + logger.info("isReady: page-url warm-up transient error %s", exc) + return False + + contents_url = f"{base}/user/{user_id}/{server_name}/api/contents/" + xsrf = _xsrf_for(session, user_id, server_name) + try: + resp = session.get(contents_url, headers={"X-XSRFToken": xsrf}, timeout=15) + except requests.RequestException as exc: + logger.info("isReady: transient error %s", exc) + return False + return resp.ok and "application/json" in resp.headers.get("content-type", "") + + +def _trigger_spawn(session: requests.Session, base: str, token: str, user_id: str, workspace_id: int, server_name: str) -> None: + # Mirrors the old browser client's triggerSpawn: chkclogin sets the Hub session cookie from + # the accessToken cookie, then /hub/spawn reads the workspaceId cookie to mount the right + # PVC. A `Session` persists both across this call and the next automatically. + session.cookies.set("kc-access", token) + session.cookies.set("accessToken", token) + session.cookies.set("workspaceId", str(workspace_id)) + + session.get(f"{base}/hub/chkclogin", params={"accessToken": token}, timeout=30) + + # If this named server is already running (e.g. the user has it open in a browser tab), + # calling /hub/spawn unconditionally makes KubeSpawner kill and recreate an already-healthy + # pod, and can race a concurrent spawn from the browser's own UI — leaving the Hub's + # server-tracking state out of sync with the actual pod (contents API 424s indefinitely even + # though the pod itself is fine). Skip the spawn call entirely when the server already answers. + if _is_ready(session, base, user_id, server_name): + return + # No accessToken param here — including it would make nginx/the hub replace all cookies with + # just accessToken, dropping workspaceId. + session.get(f"{base}/hub/spawn/{user_id}/{server_name}", timeout=30) + + +def _wait_until_ready(session: requests.Session, base: str, user_id: str, server_name: str, timeout_s: int) -> None: + deadline = time.monotonic() + timeout_s + + while time.monotonic() < deadline: + if _is_ready(session, base, user_id, server_name): + return + # Anything else (spawn-pending HTML page, 403 before the per-server cookie lands, + # 502/503 while the pod starts, a transient error) just means "not ready yet" — keep + # polling. + time.sleep(SPAWN_POLL_INTERVAL_S) + + raise ScriptRunError(f"Workspace server did not become ready within {timeout_s}s") + + +def _put_text_file(session: requests.Session, base: str, user_id: str, server_name: str, path: str, text: str) -> None: + contents_url = f"{base}/user/{user_id}/{server_name}/api/contents/{path}" + xsrf = _xsrf_for(session, user_id, server_name) + resp = session.put( + contents_url, + headers={"X-XSRFToken": xsrf, "Content-Type": "application/json"}, + json={"name": path, "path": path, "type": "file", "format": "text", "content": text}, + timeout=30, + ) + if not resp.ok: + raise ScriptRunError(f"Writing {path} into the workspace failed: HTTP {resp.status_code} — {resp.text[:300]}") + + +def _run_script(session: requests.Session, base: str, user_id: str, server_name: str, script_path: str, timeout_s: int, asset_path: str | None = None, dandiset_id: str | None = None) -> str: + xsrf = _xsrf_for(session, user_id, server_name) + kernels_url = f"{base}/user/{user_id}/{server_name}/api/kernels" + + kernel_resp = session.post( + kernels_url, + headers={"X-XSRFToken": xsrf, "Content-Type": "application/json"}, + json={"name": "python3"}, + timeout=30, + ) + if not kernel_resp.ok: + raise ScriptRunError(f"Starting a kernel failed: HTTP {kernel_resp.status_code} — {kernel_resp.text[:300]}") + kernel_id = kernel_resp.json()["id"] + + try: + return _execute_over_websocket(session, base, user_id, server_name, kernel_id, script_path, timeout_s, asset_path, dandiset_id) + finally: + # Best-effort — the output already arrived; a leaked kernel is far less bad than + # losing the result to a cleanup error. + try: + session.delete( + f"{kernels_url}/{kernel_id}", + headers={"X-XSRFToken": xsrf}, + timeout=15, + ) + except requests.RequestException: + pass + + +def _execute_over_websocket( + session: requests.Session, + base: str, + user_id: str, + server_name: str, + kernel_id: str, + script_path: str, + timeout_s: int, + asset_path: str | None = None, + dandiset_id: str | None = None, +) -> str: + session_id = uuid.uuid4().hex + msg_id = uuid.uuid4().hex + + ws_url = ( + f"{base.replace('https://', 'wss://').replace('http://', 'ws://')}" + f"/user/{user_id}/{server_name}/api/kernels/{kernel_id}/channels?session_id={session_id}" + ) + cookie_header = "; ".join(f"{c.name}={c.value}" for c in session.cookies) + + ws = websocket.create_connection( + ws_url, + header=[f"Cookie: {cookie_header}", f"Host: {_public_host()}"], + timeout=timeout_s, + ) + + # runpy rather than exec(open(...).read()) so `if __name__ == '__main__'` still fires, since + # every protocol script's real work lives under that guard. ASSET_PATH/DANDISET_ID are set + # first so resolve_asset_path() (argv, then env var, then a hardcoded fallback) picks up the + # real just-uploaded asset. sys.argv is reset to just the script name — this kernel is a real + # ipykernel process launched as `ipykernel_launcher.py -f `, so argv[1] is + # literally "-f", which resolve_asset_path() would otherwise treat as an asset-path override + # ahead of ASSET_PATH. chdir to WORKSPACE_DIR first: `_put_text_file` wrote the script via the + # Contents API, which resolves relative paths against the notebook server's configured root + # (/opt/workspace — jupyter_notebook_config.py), but a kernel's own cwd does not default to + # that root. Trailing `; None` keeps run_path()'s returned namespace dict from being + # auto-displayed as the cell's result (it would otherwise dump every name the script defined + # into both script_output and the saved .ipynb). + asset_env = f"import os; os.environ['ASSET_PATH'] = {asset_path!r}\n" if asset_path else "" + dandiset_env = f"import os; os.environ['DANDISET_ID'] = {dandiset_id!r}\n" if dandiset_id else "" + code = ( + f"{asset_env}" + f"{dandiset_env}" + f"import sys; sys.argv = [{script_path!r}]\n" + f"import os; os.chdir('/opt/workspace')\n" + f"import runpy; runpy.run_path({script_path!r}, run_name='__main__'); None" + ) + + try: + ws.send(json.dumps({ + "header": { + "msg_id": msg_id, + "session": session_id, + "username": user_id, + "msg_type": "execute_request", + "version": "5.3", + }, + "parent_header": {}, + "metadata": {}, + "content": { + "code": code, + "silent": False, + "store_history": True, + "user_expressions": {}, + "allow_stdin": False, + "stop_on_error": True, + }, + "channel": "shell", + "buffers": [], + })) + + output_parts: list[str] = [] + # nbformat-shaped outputs, built in parallel with output_parts so the same execution + # can be saved as a real, openable notebook (one code cell = the whole script) in + # addition to the plain-text string this function already returns — see + # `_notebook_document()`. Each dict here matches nbformat v4's output schema exactly. + nb_outputs: list[dict] = [] + execution_count = 1 + deadline = time.monotonic() + timeout_s + ws.settimeout(10) + + while time.monotonic() < deadline: + try: + raw = ws.recv() + except websocket.WebSocketTimeoutException: + continue + if not raw: + continue + msg = json.loads(raw) + if msg.get("parent_header", {}).get("msg_id") != msg_id: + continue + + msg_type = msg["header"]["msg_type"] + content = msg.get("content", {}) + + if msg_type == "stream": + text = content.get("text", "") + output_parts.append(text) + nb_outputs.append({"output_type": "stream", "name": content.get("name", "stdout"), "text": text}) + elif msg_type in ("execute_result", "display_data"): + text = content.get("data", {}).get("text/plain", "") + output_parts.append(text + "\n") + nb_output = {"output_type": msg_type, "data": content.get("data", {}), "metadata": content.get("metadata", {})} + if msg_type == "execute_result": + nb_output["execution_count"] = execution_count + nb_outputs.append(nb_output) + elif msg_type == "error": + traceback_lines = content.get("traceback", []) + output_parts.append("\n" + "\n".join(traceback_lines) + "\n") + nb_outputs.append({ + "output_type": "error", + "ename": content.get("ename", ""), + "evalue": content.get("evalue", ""), + "traceback": traceback_lines, + }) + elif msg_type == "status" and content.get("execution_state") == "idle": + try: + _put_notebook_file(session, base, user_id, server_name, script_path, code, nb_outputs, execution_count) + except Exception: # noqa: BLE001 — the .ipynb is a bonus artifact; the run already succeeded + logger.exception("Failed to save %s's execution as a notebook (non-fatal)", script_path) + return "".join(output_parts) + + raise ScriptRunError(f"{script_path} did not finish within {timeout_s}s") + finally: + ws.close() + + +def _notebook_document(source: str, outputs: list[dict], execution_count: int) -> dict: + """A minimal, valid nbformat v4 notebook: one code cell holding the executed code, with + the real outputs captured from that same kernel run attached to it. + """ + return { + "cells": [{ + "cell_type": "code", + "execution_count": execution_count, + "metadata": {}, + "outputs": outputs, + "source": source.splitlines(keepends=True), + }], + "metadata": { + "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, + "language_info": {"name": "python"}, + }, + "nbformat": 4, + "nbformat_minor": 5, + } + + +def _put_notebook_file( + session: requests.Session, + base: str, + user_id: str, + server_name: str, + script_path: str, + source: str, + outputs: list[dict], + execution_count: int, +) -> None: + notebook_path = os.path.splitext(script_path)[0] + ".ipynb" + contents_url = f"{base}/user/{user_id}/{server_name}/api/contents/{notebook_path}" + xsrf = _xsrf_for(session, user_id, server_name) + resp = session.put( + contents_url, + headers={"X-XSRFToken": xsrf, "Content-Type": "application/json"}, + json={ + "name": notebook_path, + "path": notebook_path, + "type": "notebook", + "format": "json", + "content": _notebook_document(source, outputs, execution_count), + }, + timeout=30, + ) + if not resp.ok: + raise ScriptRunError(f"Writing {notebook_path} into the workspace failed: HTTP {resp.status_code} — {resp.text[:300]}") diff --git a/applications/workspaces/server/workspaces/service/osbrepository/adapters/dandi_upload.py b/applications/workspaces/server/workspaces/service/osbrepository/adapters/dandi_upload.py new file mode 100644 index 000000000..ec1c2ad8e --- /dev/null +++ b/applications/workspaces/server/workspaces/service/osbrepository/adapters/dandi_upload.py @@ -0,0 +1,176 @@ +"""Uploads data INTO EMBER-DANDI (write side). + +`dandiadapter.py` in this same package is the read side — it lists/imports existing dandisets +from the *public* DANDI archive (hardcoded api.dandiarchive.org), read-only, no admin key. This +module is the write-side counterpart, against EMBER-DANDI specifically, and is the only code +holding the admin key. +""" +import mimetypes +import os + +import requests +from cloudharness.utils.secrets import get_secret + +# Set out-of-band via `kubectl create secret generic workspaces --from-literal=dandi-api-key=...` +# (deploy/values.yaml declares this secret as unmanaged — `manager: null` — specifically so +# CloudHarness never renders/overwrites it). +_DANDI_API_KEY_SECRET_NAME = "dandi-api-key" + +# EMBER-DANDI is a separate dandi-archive deployment from the public archive, with its own API +# root that its web UI never advertises. +DANDI_API_BASE = os.environ.get("WORKSPACES_DANDI_API_BASE", "https://api-dandi.emberarchive.org/api") +DANDI_DANDISET_ID = os.environ.get("WORKSPACES_DANDI_DANDISET_ID", "000533") # arc-idp-test + +# The digest key DANDI stores and echoes back in asset metadata. +_DANDI_ETAG_ALGORITHM = "dandi:dandi-etag" + + +def get_dandi_api_key() -> str: + """Never log the return value.""" + return get_secret(_DANDI_API_KEY_SECRET_NAME).strip() + + +def _headers(): + return {"Authorization": f"token {get_dandi_api_key()}"} + + +def _check(resp, what: str): + """raise_for_status() alone discards DANDI's explanation of *why* a call failed, which + turns every 4xx into a guessing game. Always surface the response body.""" + if not resp.ok: + raise RuntimeError(f"DANDI {what} failed: HTTP {resp.status_code} — {resp.text[:1000]}") + return resp + + +def get_blob_by_digest(dandi_etag: str) -> dict | None: + """POST /blobs/digest/ — returns the existing AssetBlob for this content, or None. + + DANDI separates blobs (content-addressed, deduplicated archive-wide) from assets (a path + + metadata pointing at a blob), so identical content uploaded twice is one blob with two + assets. This is how we resolve the blob when initialize_upload reports it already exists. + """ + resp = requests.post( + f"{DANDI_API_BASE}/blobs/digest/", + headers=_headers(), + json={"algorithm": _DANDI_ETAG_ALGORITHM, "value": dandi_etag}, + ) + if resp.status_code == 404: + return None + _check(resp, "get_blob_by_digest") + return resp.json() + + +def initialize_upload(size: int, dandi_etag: str) -> dict: + """POST /uploads/initialize/ — returns {upload_id, parts: [{part_number, size, upload_url}]}. + + Reserves space for content only; the asset path is not part of this call and is supplied + later, at `register_asset`. `dandi_etag` must already be in DANDI's S3-multipart format + (`<32-hex>-`), computed client-side since bytes never reach here. + + A 409 here is not a failure: it means this exact content is already in the archive, so there + is nothing to transfer. We resolve the existing blob and return `{"blob_id": ...}` with no + parts, letting the caller skip straight to registering a new asset against it. + """ + resp = requests.post( + f"{DANDI_API_BASE}/uploads/initialize/", + headers=_headers(), + json={ + "contentSize": size, + "dandiset": DANDI_DANDISET_ID, + "digest": {"algorithm": _DANDI_ETAG_ALGORITHM, "value": dandi_etag}, + }, + ) + if resp.status_code == 409: + blob = get_blob_by_digest(dandi_etag) + if not blob: + raise RuntimeError( + f"DANDI reported the blob already exists (409) but no blob matches digest {dandi_etag}" + ) + return {"upload_id": None, "parts": [], "blob_id": blob["blob_id"]} + + _check(resp, "initialize_upload") + return resp.json() + + +def complete_upload(upload_id: str, parts: list) -> dict: + """POST /uploads/{upload_id}/complete/, then POST the returned presigned S3 + CompleteMultipartUpload request ourselves — the browser never sees this step. + + `parts` is [{part_number, size, etag}], `etag` being what S3 returned for each part PUT. + Returns the completion response body from S3 (not currently parsed further). + """ + resp = requests.post( + f"{DANDI_API_BASE}/uploads/{upload_id}/complete/", + headers=_headers(), + json={"parts": parts}, + ) + _check(resp, "complete_upload") + completion = resp.json() + + s3_resp = requests.post( + completion["complete_url"], + data=completion["body"], + headers={"Content-Type": "text/xml"}, + ) + _check(s3_resp, "S3 complete-multipart") + return {"status": s3_resp.status_code} + + +def validate_upload(upload_id: str) -> dict: + """POST /uploads/{upload_id}/validate/ — returns AssetBlob: {blob_id, etag, size, sha256}.""" + resp = requests.post(f"{DANDI_API_BASE}/uploads/{upload_id}/validate/", headers=_headers()) + _check(resp, "validate_upload") + return resp.json() + + +def register_asset(path: str, blob_id: str) -> dict: + """POST the completed blob into the dandiset's draft version as a new asset. + + The collection endpoint only supports GET/POST — PUT is for updating a specific existing + asset at `.../assets/{asset_id}/`. AssetRequest only requires `metadata`, and `path` goes + *inside* it rather than being a top-level field. + + `schemaKey` and `encodingFormat` are the only two fields the dandi-schema `Asset` model + requires beyond what DANDI backfills itself (contentSize, digest, id, contentUrl) — leaving + either out is what was producing "'X' is a required property" validation errors on every + asset. Real subject/session metadata is still not populated here. + """ + encoding_format, _ = mimetypes.guess_type(path) + resp = requests.post( + f"{DANDI_API_BASE}/dandisets/{DANDI_DANDISET_ID}/versions/draft/assets/", + headers=_headers(), + json={ + "metadata": { + "path": path, + "schemaKey": "Asset", + "encodingFormat": encoding_format or "application/octet-stream", + }, + "blob_id": blob_id, + }, + ) + if resp.status_code == 409: + # Already registered at this path — almost always a retry after a later step (the + # workspace-attach calls) failed. Reuse it rather than failing, so finalize is idempotent. + existing = get_asset_by_path(path) + if existing: + return existing + _check(resp, f"register_asset(path={path!r})") + return resp.json() + + +def get_asset_by_path(path: str) -> dict | None: + """GET the asset at an exact path in the draft version, or None.""" + resp = requests.get( + f"{DANDI_API_BASE}/dandisets/{DANDI_DANDISET_ID}/versions/draft/assets/", + headers=_headers(), + params={"path": path}, + ) + _check(resp, f"get_asset_by_path(path={path!r})") + for asset in resp.json().get("results", []): + if asset.get("path") == path: + return asset + return None + + +def dandiset_url() -> str: + return f"https://dandi.emberarchive.org/dandiset/{DANDI_DANDISET_ID}/draft"