情况:我们在 Azure 数据湖分析中创建了数据库“CLSTrackOMeter”和表“Customer_Information”。
Customer_Information，将图像路径存储在暂存文件夹中(目前我已将源图像路径硬编码在类库中)。
议程:使用 CustInfo 中的值将数据上传到 Azure 数据湖存储“Customer_Image”文件夹
尝试过的解决方案 - 创建usql类库，使用.net sdk上传文件(能够在控制台应用程序中执行该类库)，并部署在azure数据湖存储中。 - 添加了新的USQL脚本并引用了该类库
- 在usql脚本的cs文件中调用类库
类库代码
using Microsoft.Analytics.Interfaces;
using Microsoft.Analytics.Types.Sql;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.Azure.Management.DataLake.Store;
using Microsoft.Azure.Management.DataLake.Store.Models;
using Microsoft.IdentityModel.Clients.ActiveDirectory;
using Microsoft.Rest.Azure.Authentication;
using Microsoft.Azure.Management.DataLake.StoreUploader;
using System.Threading;
using System.Diagnostics;
using System.Collections;
using System.Threading.Tasks;
namespace USQLCSharpProject1
{
public static class Program
{
private static DataLakeStoreAccountManagementClient _adlsClient;
private static DataLakeStoreFileSystemManagementClient _adlsFileSystemClient;
private static string _adlsAccountName;
private static string _resourceGroupName;
private static string _location;
private static string _subId;
//private static void Main(string[] args)
public static string UploadFileWithS2S_WithClientSecret(string s)
{
try
{
_adlsAccountName = "<DATA-LAKE-STORE-NAME>"; // TODO: Replace this value with the name of your existing Data Lake Store account.
_resourceGroupName = "<RESOURCE-GROUP-NAME>"; // TODO: Replace this value with the name of the resource group containing your Data Lake Store account.
_location = "East US 2";
_subId = "<SUBSCRIPTION-ID>";
string localFolderPath = @"D:\Harry\PSR\study\TEST"; // TODO: Make sure this exists and can be overwritten.
string localFilePath = Path.Combine(localFolderPath, "fileTwo.txt"); // TODO: Make sure this exists and can be overwritten.
string remoteFolderPath = "/Samples/OUTPUT";
//string remoteFilePath = Path.Combine(remoteFolderPath, "file.txt");
// Service principal / appplication authentication with client secret / key
// Use the client ID of an existing AAD "Web App" application.
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
var domain = "<AAD-directory-domain>";
var webApp_clientId = "<AAD-application-clientid>";
var clientSecret = "<AAD-application-client-secret>";
var clientCredential = new ClientCredential(webApp_clientId, clientSecret);
var creds = ApplicationTokenProvider.LoginSilentAsync(domain,clientCredential).Result;
// Create client objects and set the subscription ID
_adlsClient = new DataLakeStoreAccountManagementClient(creds) { SubscriptionId = _subId };
_adlsFileSystemClient = new DataLakeStoreFileSystemManagementClient(creds);
var parameters = new UploadParameters(localFolderPath, remoteFolderPath, _adlsAccountName, isOverwrite: true); // the default maxSegmentLength is 256M, we can set by ourself.
var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient);
var uploader = new DataLakeStoreUploader(parameters, frontend);
uploader.Execute();
return s;
}
catch (Exception ex)
{
return "";
}
}
}
}
Usql代码
USE CLSTrackOMeter;
REFERENCE ASSEMBLY USQLCSharpProject1;
@result =
SELECT USQLUploadFile.myFirstClass.myFirstFunction(AgeGender)AS myFirstFunction_CB
FROM CLSTrackOMeter.dbo.Customer_Information;
USQL cs文件代码
using Microsoft.Analytics.Interfaces;
using Microsoft.Analytics.Types.Sql;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace USQLUploadFile
{
public class myFirstClass
{
public static string myFirstFunction(string s)
{
try
{
string aa = USQLCSharpProject1.Program.UploadFileWithS2S_WithClientSecret("rajni");
return aa;
}
catch (Exception ex)
{
return "";
}
}
}
}
项目图片
错误图片
使用 PROCESS 表达式时出错
PROCESS 表达式的 USQL 代码
USE CLSTrackOMeter;
REFERENCE ASSEMBLY USQLCSharpProject1;
@result = SELECT AgeGender
FROM CLSTrackOMeter.dbo.Customer_Information;
@rs=
PROCESS @result
PRODUCE AgeGender
USING new USQLUploadFile.myFirstClass();
OUTPUT @rs
TO "/output/Harry.csv"
USING Outputters.Csv();
USQL CS 文件处理表达式代码
using Microsoft.Analytics.Interfaces;
using Microsoft.Analytics.Types.Sql;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace USQLUploadFile
{
[SqlUserDefinedProcessor]
public class myFirstClass : IProcessor
{
public override IRow Process(IRow input, IUpdatableRow output)
{
try
{
string AgeGender = input.Get<string>("AgeGender");
//USQLCSharpProject1.Class1 obj = new ClassLibrary1.Class1();
//string aa = USQLCSharpProject1.Program.UploadFileWithS2S_WithClientSecret("rajni");
//return aa;
string aa=USQLCSharpProject1.Program.UploadFileWithS2S_WithClientSecret("AgeGender");
output.Set<string>("AgeGender", AgeGender);
return output.AsReadOnly();
//return obj.newTest(s);
}
catch (Exception ex)
{
return null;
}
}
}
}
添加:
注册 .Net SDK 库并在 USQL 中引用它们后
正在提交作业，在输出中显示以下文本
System.InvalidOperationException: Collection was modified; enumeration operation may not execute.
at System.ThrowHelper.ThrowInvalidOperationException(ExceptionResource resource)
at System.Collections.Generic.List`1.Enumerator.MoveNextRare()
at System.Collections.Generic.List`1.Enumerator.MoveNext()
at Microsoft.Cosmos.ScopeStudio.VsExtension.ProjectSystem.ScopeProjectNode.get_ReferenceInfoList()
at Microsoft.Cosmos.ScopeStudio.BusinessObjects.Common.ScriptToProjectTable.GetProjectReferenceList(String scriptFilePath)
at Microsoft.Cosmos.ScopeStudio.UserInterface.SQLIP.BaseSubmissionViewModel`1.GetScriptContentsWithReference(ProductFunctionType productType)
at Microsoft.Cosmos.ScopeStudio.UserInterface.SQLIP.DataLakeJobSubmissionViewModel`1.DoJobSubmission()
添加其他引用图片
在 ADLS 中注册程序集
最佳答案
我对你想要实现的目标有点困惑。您是否尝试从 U-SQL 用户定义运算符中调用 Azure SDK？这是行不通的，因为 U-SQL 容器不允许调用 Web 服务 API，包括数据湖 REST API。
关于azure - 我们不能使用USQL自定义代码和usql上传文档/图像吗？，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/43896225/