c# - 从 blob 存储中获取资源文件并添加到 azure 批处理中的任务

Question

我是 Azure 和 Azure Batch 服务的新手。我创建了一个简单的 .NET 应用程序，它应该在某个输入文件上运行一些代码。这就是我开始创建作业、任务并添加随后应由我的应用程序包处理的资源文件的方式。

1.) 创建 BlobClient 等:

        // Create the blob client, for use in obtaining references to blob storage containers
        CloudBlobClient blobClient = CreateCloudBlobClient(StorageAccountName, StorageAccountKey);

        // Use the blob client to create the input container in Azure Storage 
        const string inputContainerName = "modelinput";

        CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);

        container.CreateIfNotExistsAsync().Wait();

2.) 这里上传了一个放置在应用程序目录中的文件，并将其添加到资源文件列表中:

         List<ResourceFile> inputFiles = new List<ResourceFile>();

//upload the file that should be processed and add to resourcefiles
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, "myinputfile.xml"))

3.) 创建批处理作业和任务

    BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);

        using (BatchClient batchClient = BatchClient.Open(cred))
        {

            Console.WriteLine("Creating job [{0}]...", JobId);


                CloudJob job = batchClient.JobOperations.CreateJob();
                job.Id = JobId;
                job.PoolInformation = new PoolInformation { PoolId = PoolId }; 

                job.Commit();

        List<CloudTask> tasks = new List<CloudTask>();

        string taskId = String.Format("Task{0}", 0);
        string inputFilename = inputFiles[0].FilePath;

        //set the commandline 
        string taskCommandLine = String.Format("cmd /c %AZ_BATCH_APP_PACKAGE_DEVBATCHAPP%\\batchapp.exe {0}", inputFilename);

        CloudTask task = new CloudTask(taskId, taskCommandLine)

         //add my resourcefiles to the task
        task.ResourceFiles = new List<ResourceFile> { inputFiles[0] }; 

        task.ApplicationPackageReferences = new List<ApplicationPackageReference>{new ApplicationPackageReference{ApplicationId = "devbatchapp",Version = "0.0.1"}};

        tasks.Add(task);
     }

如果我现在运行任务，一切都会完美运行。不过现在我开发了一个小的 ASP.NET Razor Pages 应用程序，用户可以通过它选择并上传文件。由于我现在只能使用文件流来上传我的文件，因此我必须将 2.) 更改为类似的内容，主要遵循文档 here :

            var filePath = Path.GetTempFileName();

            var file = Path.Combine(_environment.ContentRootPath, "uploads", filePath); 

        string containerSasToken = container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
        {
            SharedAccessStartTime = DateTime.UtcNow,
            SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1),
            Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read
        });
        string containerSas = container.Uri.AbsoluteUri + containerSasToken; 

        using (var stream = new FileStream(file, FileMode.Create))
        {
            await Upload.CopyToAsync(stream);
            stream.Seek(0, SeekOrigin.Begin);
            var blobstore = container.GetBlockBlobReference("modelinput");
            await blobstore.UploadFromStreamAsync(stream);

            containerSas = blobstore.Uri.AbsoluteUri + containerSasToken;
            inputFiles = new List<ResourceFile> { 
            ResourceFile.FromStorageContainerUrl(containerSas, "myinput") };

        }

其余代码基本保持不变。但是现在当我尝试运行该任务时，我收到了 BlobDownloadMiscError 。在批处理资源管理器中对此进行分析时，我发现资源文件的 URL 显然是错误的，如果我手动将其添加到它可以工作的任务中。有人可以帮助我吗？如何获取资源文件的正确来源并将其添加到我的任务中？

Answer 1

根据我的测试，您可能没有使用正确的权限。更多详情请引用document 。此外，请确保您的存储帐户已链接到您的批量帐户。

For container access, you must have both Read and List permissions, whereas with blob access, you only need Read permission.

我的代码如下

string containerSasToken = container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
    SharedAccessStartTime = DateTime.UtcNow,
    SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1),
    Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List
});

string containerSasUrl = String.Format("{0}{1}", container.Uri, containerSasToken);
var inputFiles = new List<ResourceFile> { };
var file = ResourceFile.FromStorageContainerUrl(containerSasUrl,"test");
inputFiles.Add(file);
Console.WriteLine(inputFiles.Count);

// Get a Batch client using account creds

BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);

using (BatchClient batchClient = BatchClient.Open(cred))
{
    Console.WriteLine("getting pool [{0}]...", PoolId);


    batchClient.PoolOperations.GetPool(PoolId);
    
    // Create a Batch job
    Console.WriteLine("Creating job [{0}]...", JobId);

    try
    {
        CloudJob job = batchClient.JobOperations.CreateJob();
        job.Id = JobId;
        job.PoolInformation = new PoolInformation { PoolId = PoolId };

        job.Commit();
    }
    catch (BatchException be)
    {
        // Accept the specific error code JobExists as that is expected if the job already exists
        if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
        {
            Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
        }
        else
        {
            throw; // Any other exception is unexpected
        }
    }

    // Create a collection to hold the tasks that we'll be adding to the job

    Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);

    List<CloudTask> tasks = new List<CloudTask>();

    // Create each of the tasks to process one of the input files. 

    for (int i = 0; i < inputFiles.Count; i++)
    {
        string taskId = String.Format("Task{0}", i);
        string inputFilename = inputFiles[i].FilePath;
        string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);

        CloudTask task = new CloudTask(taskId, taskCommandLine);
        task.ResourceFiles = new List<ResourceFile> { inputFiles[i] };
        tasks.Add(task);
    }

    // Add all tasks to the job.
    batchClient.JobOperations.AddTask(JobId, tasks);


    // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.

    TimeSpan timeout = TimeSpan.FromMinutes(30);
    Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);

    IEnumerable<CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);

    batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);

    Console.WriteLine("All tasks reached state Completed.");

    // Print task output
    Console.WriteLine();
    Console.WriteLine("Printing task output...");
}