Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 14 additions & 28 deletions app/Http/Controllers/SubmissionController.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,12 @@
use Exception;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Response;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Validator;
use League\Flysystem\UnableToMoveFile;
use League\Flysystem\UnableToReadFile;
use League\Flysystem\UnableToWriteFile;
use Symfony\Component\HttpFoundation\Exception\BadRequestException;
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
use Symfony\Component\HttpKernel\Exception\HttpException;

Expand Down Expand Up @@ -73,6 +71,7 @@ private function failProcessing(?string $filename, int $outResponseCode, string
}

/**
* @throws BadRequestException
* @throws BadSubmissionException
*/
private function submitProcess(): Response
Expand All @@ -83,7 +82,7 @@ private function submitProcess(): Response
$responseMessage = '';
$projectname = request()->string('project', '');

if (strlen($projectname) === 0) {
if (!request()->filled('project')) {
Log::info('Rejected submission with no project name');
$this->failProcessing(null, Response::HTTP_BAD_REQUEST, 'No project name provided.');
}
Expand All @@ -99,41 +98,32 @@ private function submitProcess(): Response
$this->failProcessing(null, Response::HTTP_BAD_REQUEST, "Invalid project name: $projectname");
}

$expected_md5 = request()->query('MD5', '');
if ($expected_md5 !== '' && !preg_match('/^[a-f0-9]{32}$/i', $expected_md5)) {
$expected_md5 = request()->query->getString('MD5');
if (request()->filled('MD5') && !SubmissionUtils::isValidMD5($expected_md5)) {
Log::info("Rejected submission with invalid hash '$expected_md5' for project $projectname");
$this->failProcessing(null, Response::HTTP_BAD_REQUEST, "Provided md5 hash '{$expected_md5}' is improperly formatted.");
}

// Get auth token (if any).
$authtoken = request()->bearerToken();
$authtoken_hash = $authtoken === null || $authtoken === '' ? '' : AuthTokenService::hash($authtoken);
$authtoken_hash = AuthTokenService::hash(request()->bearerToken());

// Check that the md5sum of the file matches what we were told to expect.
$fp = request()->getContent(true);
if (strlen($expected_md5) > 0) {
if (request()->filled('MD5') && !SubmissionUtils::isFileMD5Correct($fp, $expected_md5)) {
$md5sum = SubmissionUtils::hashFileHandle($fp, 'md5');
if ($md5sum !== $expected_md5) {
Log::info("Rejected submission because hash '$md5sum' does not match the expected hash '$expected_md5' for project $projectname");
$this->failProcessing(null, Response::HTTP_BAD_REQUEST, "md5 mismatch. expected: {$expected_md5}, received: {$md5sum}");
}
Log::info("Rejected submission because hash '$md5sum' does not match the expected hash '$expected_md5' for project $projectname");
$this->failProcessing(null, Response::HTTP_BAD_REQUEST, "md5 mismatch. expected: {$expected_md5}, received: {$md5sum}");
}

// Save the incoming file in the inbox directory.
$filename = SubmissionUtils::xmlFilename((string) $projectname, $authtoken_hash, $expected_md5);
try {
Storage::put("inbox/{$filename}", $fp);
} catch (UnableToWriteFile $e) {
report($e);
if (!SubmissionUtils::storeInInbox($filename, $fp)) {
$this->failProcessing($filename, Response::HTTP_INTERNAL_SERVER_ERROR, 'Failed to save submission file.');
}

// Check if we can connect to the database before proceeding any further.
if (!SystemUtils::isDatabaseOnline()) {
// Write a marker file so we know to process these files when the DB comes back up.
if (!Storage::exists('DB_WAS_DOWN')) {
Storage::put('DB_WAS_DOWN', '');
}
SubmissionUtils::markDeferredSubmissions();
$statusarray['status'] = 'OK';
$statusarray['message'] = 'Database is unavailable.';
return self::displayXMLReturnStatus($statusarray);
Expand Down Expand Up @@ -193,15 +183,15 @@ private function submitProcess(): Response

// Check if CTest provided us enough info to assign a buildid.
$buildid = null;
if (request()->has('build') && request()->has('site') && request()->has('stamp')) {
if (request()->filled('build') && request()->filled('site') && request()->filled('stamp')) {
$build = new Build();
$build->Name = pdo_real_escape_string(request()->query('build'));
$build->ProjectId = $this->project->Id;
$build->SetStamp(pdo_real_escape_string(request()->query('stamp')));
$build->StartTime = gmdate(FMT_DATETIME);
$build->SubmitTime = $build->StartTime;

if (request()->has('subproject')) {
if (request()->filled('subproject')) {
$build->SetSubProject(pdo_real_escape_string(request()->query('subproject')));
}

Expand Down Expand Up @@ -229,11 +219,7 @@ private function submitProcess(): Response
fclose($fp);
unset($fp);

// Check for marker file to see if we need to queue deferred submissions.
if (Storage::exists('DB_WAS_DOWN')) {
Storage::delete('DB_WAS_DOWN');
Artisan::call('submission:queue');
}
SubmissionUtils::queueDeferredSubmissions();

$statusarray['message'] = '';
if ($responseMessage !== '') {
Expand Down
47 changes: 47 additions & 0 deletions app/Utils/SubmissionUtils.php
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@
use CDash\Database;
use CDash\Model\Build;
use CDash\Model\BuildUpdate;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use League\Flysystem\UnableToWriteFile;

class SubmissionUtils
{
Expand Down Expand Up @@ -331,6 +334,50 @@ public static function hashFileHandle(mixed $filehandle, string $algo): string
return hash_final($hashContext);
}

/** Check whether a string is a properly formatted md5 hash */
public static function isValidMD5(string $md5): bool
{
return preg_match('/^[a-f0-9]{32}\z/i', $md5) === 1;
}

/** Check whether the md5 of an open file handle matches the expected hash, ignoring case */
public static function isFileMD5Correct(mixed $filehandle, string $expected_md5): bool
{
return strcasecmp(self::hashFileHandle($filehandle, 'md5'), $expected_md5) === 0;
}

/**
* Save a submission file to the inbox directory
*
* @param resource|string $contents
*/
public static function storeInInbox(string $filename, mixed $contents): bool
{
try {
return Storage::put("inbox/{$filename}", $contents);
} catch (UnableToWriteFile $e) {
report($e);
return false;
}
}

/** Record that the inbox holds submissions received while the database was unavailable */
public static function markDeferredSubmissions(): void
{
if (!Storage::exists('DB_WAS_DOWN')) {
Storage::put('DB_WAS_DOWN', '');
}
}

/** Queue any submissions received while the database was unavailable */
public static function queueDeferredSubmissions(): void
{
if (Storage::exists('DB_WAS_DOWN')) {
Storage::delete('DB_WAS_DOWN');
Artisan::call('submission:queue');
}
}

/** Generate the inbox filename for a CTest XML submission */
public static function xmlFilename(string $projectname, string $token_hash, string $md5): string
{
Expand Down
64 changes: 27 additions & 37 deletions app/Utils/UnparsedSubmissionProcessor.php
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,15 @@
use App\Jobs\ProcessSubmission;
use App\Models\BuildFile;
use App\Models\PendingSubmissions;
use App\Models\Project as EloquentProject;
use App\Models\Site;
use App\Rules\ProjectNameRule;
use App\Services\AuthTokenService;
use CDash\Model\Build;
use CDash\Model\Project;
use Closure;
use Exception;
use Illuminate\Http\JsonResponse;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
Expand Down Expand Up @@ -98,10 +99,7 @@ public function postSubmit(): JsonResponse
$uuid = Str::uuid()->toString();
$this->serializeBuildMetadata($uuid);

// Write a marker file so we know to process these files when the DB comes back up.
if (!Storage::exists('DB_WAS_DOWN')) {
Storage::put('DB_WAS_DOWN', '');
}
SubmissionUtils::markDeferredSubmissions();

// Respond with success even though the database is down so that CTest will
// proceed to upload the data file.
Expand Down Expand Up @@ -145,17 +143,15 @@ public function parseBuildMetadata(): void
public function initializeBuild(): JsonResponse
{
// Retrieve this project from the database.
$project_row = DB::table('project')
->where('name', $this->projectname)
->first();
$eloquent_project = EloquentProject::firstWhere('name', $this->projectname);

if (!$project_row) {
if ($eloquent_project === null) {
abort(Response::HTTP_NOT_FOUND, 'Project does not exist');
}
$projectid = $project_row->id;
$projectid = $eloquent_project->id;

// Check if this submission requires a valid authentication token.
if (($this->token || $project_row->authenticatesubmissions) && !AuthTokenService::check($this->token, $projectid)) {
if (($this->token || $eloquent_project->authenticatesubmissions) && !AuthTokenService::check($this->token, $projectid)) {
abort(Response::HTTP_FORBIDDEN, 'Forbidden');
}

Expand All @@ -168,7 +164,7 @@ public function initializeBuild(): JsonResponse
// Add the build.
$this->build = new Build();

$this->build->ProjectId = get_project_id($this->projectname);
$this->build->ProjectId = $projectid;
$this->build->StartTime = gmdate(FMT_DATETIME, $this->starttime);
$this->build->EndTime = gmdate(FMT_DATETIME, $this->endtime);
$this->build->SubmitTime = gmdate(FMT_DATETIME);
Expand Down Expand Up @@ -244,8 +240,6 @@ public function putSubmitFile(): JsonResponse
abort(Response::HTTP_BAD_REQUEST, $validator->errors()->first());
}
$this->projectname = $this->project->Name;

$this->inboxdatafilename = 'inbox/' . SubmissionUtils::dataFilename($this->projectname, $this->token, $this->type, $this->buildid, $this->md5, $ext);
} else {
// Get project name from build metadata file on disk.
$projectname = null;
Expand All @@ -264,18 +258,19 @@ public function putSubmitFile(): JsonResponse
abort(Response::HTTP_NOT_FOUND, 'Build not found');
}
$this->projectname = $projectname;
$this->inboxdatafilename = 'inbox/' . SubmissionUtils::dataFilename($this->projectname, $this->token, $this->type, $this->buildid, $this->md5, $ext);
$this->serializeDataFileParameters();
}

if (!Storage::exists('DB_WAS_DOWN')) {
Storage::put('DB_WAS_DOWN', '');
}
$filename = SubmissionUtils::dataFilename($this->projectname, $this->token, $this->type, $this->buildid, $this->md5, $ext);
$this->inboxdatafilename = 'inbox/' . $filename;

if (!$db_up) {
$this->serializeDataFileParameters();
SubmissionUtils::markDeferredSubmissions();
}

// Write this file to the inbox directory.
$handle = request()->getContent(true);
if (!Storage::put($this->inboxdatafilename, $handle)) {
abort(Response::HTTP_INTERNAL_SERVER_ERROR, "Cannot open file ($this->inboxdatafilename)");
if (!SubmissionUtils::storeInInbox($filename, request()->getContent(true))) {
abort(Response::HTTP_INTERNAL_SERVER_ERROR, 'Failed to save submission file.');
}

if (!$db_up) {
Expand All @@ -287,14 +282,9 @@ public function putSubmitFile(): JsonResponse
// THis function will throw an exception if invalid data provided
$this->populateBuildFileRow();

$filename = str_replace('inbox/', '', $this->inboxdatafilename);
ProcessSubmission::dispatch($filename, $this->project->Id, $this->build->Id, $this->md5);

// Check for marker file to see if we need to queue deferred submissions.
if (Storage::exists('DB_WAS_DOWN')) {
Storage::delete('DB_WAS_DOWN');
Artisan::call('submission:queue');
}
SubmissionUtils::queueDeferredSubmissions();

return response()->json($response_array);
}
Expand Down Expand Up @@ -325,8 +315,9 @@ public function populateBuildFileRow(): void
}

// Check that the md5sum of the file matches what we were expecting.
$md5sum = SubmissionUtils::hashFileHandle(Storage::readStream($this->inboxdatafilename), 'md5');
if ($md5sum != $this->md5) {
$stream = Storage::readStream($this->inboxdatafilename);
if (!SubmissionUtils::isFileMD5Correct($stream, $this->md5)) {
$md5sum = SubmissionUtils::hashFileHandle($stream, 'md5');
Storage::delete($this->inboxdatafilename);
$buildfile->delete();
abort(Response::HTTP_BAD_REQUEST, "md5 mismatch. expected: {$this->md5}, received: $md5sum");
Expand All @@ -350,7 +341,11 @@ public function parseDataFileParameters(): void
$validator = Validator::make(request()->query(), [
'buildid' => 'required',
'type' => ['required', Rule::enum(UnparsedSubmissionType::class)],
'md5' => ['required', 'regex:/^[a-f0-9]{32}$/i'],
'md5' => ['required', function (string $attribute, mixed $value, Closure $fail): void {
if (!is_string($value) || !SubmissionUtils::isValidMD5($value)) {
$fail('The :attribute format is invalid.');
}
}],
'filename' => 'required',
]);

Expand Down Expand Up @@ -455,11 +450,6 @@ private function getAuthTokenHash(): void
if ($this->token) {
return;
}
$token = request()->bearerToken();
if ($token) {
$this->token = AuthTokenService::hash($token);
} else {
$this->token = '';
}
$this->token = AuthTokenService::hash(request()->bearerToken());
}
}
Loading
Loading