Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Created dotnet2.0 Builder Image and Added /v2/specialized Endpoint to dotnet2.0 Envrionment (#1001)

Browse Source
This commit is contained in:
Paras Patidar
2019-05-15 16:35:55 +08:00
committed by Ta-Ching Chen
parent 56fa8fe766
commit 4632269314
35 changed files with 1973 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+4
View File
@@ -16,3 +16,7 @@ environments/php7/vendor/
.DS_Store
vendor/
local/
environments/dotnet20/Builder/.vs/
environments/dotnet20/.vs/
environments/dotnet20/obj/
environments/dotnet20/bin/
environments/dotnet20/.dockerignore
+1
View File
@@ -0,0 +1 @@
builder
environments/dotnet20/ExecutorModule.cs
+98 -1
View File
@@ -4,11 +4,16 @@ using System.Collections.Generic;
using Nancy;
using System.IO;
using System;
using Nancy.IO;
using Fission.DotNetCore.Utilty;
using Fission.DotNetCore.Model;
using Nancy.Extensions;
namespace Fission.DotNetCore
{
public class ExecutorModule : NancyModule
{
private static string PackagePath = string.Empty;
#if DEBUG
private const string CODE_PATH = "/tmp/func.cs";
#else
@@ -20,6 +25,7 @@ namespace Fission.DotNetCore
public ExecutorModule()
{
Post("/specialize", args => Specialize());
Post("/v2/specialize", args => Specializev2());
Get("/", _ => Run());
Post("/", _ => Run());
Put("/", _ => Run());
@@ -28,6 +34,94 @@ namespace Fission.DotNetCore
Delete("/", _ => Run());
}
private object Specializev2()
{
Console.WriteLine("Call Reached at /v2/specialize");
try
{
var errors = new List<string>();
var oinfo = new List<string>();
var _request = Request;
var _body = Request.Body;
// Request.Body.Position = 0; use it only if requst has already been read before that
var _requestBodystring = RequestStream.FromStream(Request.Body).AsString();
Console.WriteLine($"Request received by endpoint from builder : {_requestBodystring}");
BuilderRequest builderRequest = EnvironmentHelper.Instance.GetBuilderRequest(_requestBodystring);
if (builderRequest == null)
{
Console.WriteLine("Error : Unbale to parse builder request!!");
throw new Exception("Error : Unbale to parse builder request!!");
}
string functionPath = string.Empty;
// functionPath = Path.Combine(builderRequest.filepath, $"{builderRequest.functionName}.cs");
PackagePath = builderRequest.filepath;
//following will enable us to skip --entrypoint flag during function creation
if (!string.IsNullOrWhiteSpace(builderRequest.functionName))
{
functionPath = Path.Combine(builderRequest.filepath, $"{builderRequest.functionName}.cs");
}
else
{
functionPath = Path.Combine(builderRequest.filepath, EnvironmentHelper.Instance.environmentSettings.functionBodyFileName);
}
Console.WriteLine($"Going to read function body from path : {functionPath}");
if (File.Exists(functionPath))
{
var code = File.ReadAllText(functionPath);
try
{
FissionCompiler fissionCompiler = new FissionCompiler(builderRequest.filepath);
_userFunc = fissionCompiler.Compilev2(code, out errors, out oinfo);
}
catch (Exception ex)
{
Console.WriteLine($"Error getting _userFunc :{ex.Message} , Trace : {ex.StackTrace}");
}
if (_userFunc == null)
{
var errstr = string.Join(Environment.NewLine, errors);
_logger.WriteError(errstr);
Console.WriteLine($"Error _userFunc is null :{errstr}");
var response = (Response)errstr;
response.StatusCode = HttpStatusCode.InternalServerError;
return response;
}
else
{
//try to retrun few details
var infostr = string.Join(Environment.NewLine, oinfo);
_logger.WriteInfo(infostr);
var response = (Response)infostr;
response.StatusCode = HttpStatusCode.OK;
return response;
}
}
else
{
var errstr = $"Unable to locate code at '{functionPath}'";
_logger.WriteError(errstr);
var response = (Response)errstr;
response.StatusCode = HttpStatusCode.InternalServerError;
return response;
}
}
catch (Exception ex)
{
Console.WriteLine($"Exception occured {ex.Message} | {ex.StackTrace}");
var errstr = $"Exception occured {ex.Message} | {ex.StackTrace}";
_logger.WriteError(errstr);
var response = (Response)errstr;
response.StatusCode = HttpStatusCode.InternalServerError;
return response;
}
}
private object Specialize()
{
var errors = new List<string>();
@@ -67,7 +161,10 @@ namespace Fission.DotNetCore
try
{
return _userFunc.Invoke(FissionContext.Build(Request, new Logger()));
var context = FissionContext.Build(Request, new Logger());
//set the package path ,as that will be required to get appsetting files from package
context.PackagePath = PackagePath;
return _userFunc.Invoke(context);
}
catch (Exception e)
{
environments/dotnet20/FissionCompiler.cs
+157 -2
View File
@@ -1,8 +1,11 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.Loader;
using Fission.DotNetCore.Model;
using Fission.DotNetCore.Utilty;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Emit;
@@ -12,6 +15,12 @@ namespace Fission.DotNetCore.Compiler
//adapted from this article http://www.tugberkugurlu.com/archive/compiling-c-sharp-code-into-memory-and-executing-it-with-roslyn
class FissionCompiler
{
string packagepath = string.Empty;
FunctionSpecification functionSpecification = null;
public FissionCompiler(string _packagePath)
{
this.packagepath = _packagePath;
}
public static Function Compile(string code, out List<string> errors)
{
errors = new List<string>();
@@ -64,12 +73,158 @@ namespace Fission.DotNetCore.Compiler
ms.Seek(0, SeekOrigin.Begin);
Assembly assembly = AssemblyLoadContext.Default.LoadFromStream(ms);
var type = assembly.GetType("FissionFunction");
//support for Namespace , as well as backward compatibility for existing functions
var type = assembly.GetTypes().FirstOrDefault(x => x.Name.EndsWith("FissionFunction"));
var info = type.GetMember("Execute").First() as MethodInfo;
return new Function(assembly, type, info);
}
}
return null;
}
public Function Compilev2(string code, out List<string> errors, out List<string> oinfo)
{
errors = new List<string>();
oinfo = new List<string>();
#region syntext tree and default refrence build
SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(code);
string assemblyName = Path.GetRandomFileName();
var coreDir = Directory.GetParent(typeof(Enumerable).GetTypeInfo().Assembly.Location);
Console.WriteLine("Adding core refrences !!");
List<MetadataReference> references = new List<MetadataReference>
{
MetadataReference.CreateFromFile(coreDir.FullName + Path.DirectorySeparatorChar + "mscorlib.dll"),
MetadataReference.CreateFromFile(coreDir.FullName + Path.DirectorySeparatorChar + "netstandard.dll"),
MetadataReference.CreateFromFile(typeof(object).GetTypeInfo().Assembly.Location),
MetadataReference.CreateFromFile(Assembly.GetEntryAssembly().Location),
MetadataReference.CreateFromFile(typeof(System.Runtime.Serialization.Json.DataContractJsonSerializer).GetTypeInfo().Assembly.Location)
};
Console.WriteLine("Adding parent assembaly based refrences !!");
foreach (var referencedAssembly in Assembly.GetEntryAssembly().GetReferencedAssemblies())
{
var assembly = Assembly.Load(referencedAssembly);
references.Add(MetadataReference.CreateFromFile(assembly.Location));
}
#endregion
#region load function specs based dlls
Console.WriteLine($"going to get function specification...");
//load all available dlls from deployment folder in dllinfo object
functionSpecification = EnvironmentHelper.Instance.GetFunctionSpecs(packagepath);
Console.WriteLine($"going to get package dlls...");
//iterate and all all libraries mentioned
foreach (var library in functionSpecification.libraries)
{
string dllCompletePath = Path.Combine(packagepath, library.path).GetrelevantPathAsPerOS();
references.Add(MetadataReference.CreateFromFile(dllCompletePath));
Console.WriteLine($"refered folder based dll : {dllCompletePath} from package {library.nugetPackage}");
}
Console.WriteLine($"refered all available dlls!!");
oinfo.Add("refered all available dlls!!");
#endregion
#region dynamic resolve handeler registration
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.AssemblyResolve += CurrentDomain_AssemblyResolve;
#endregion
#region function compile
Console.WriteLine($"Trying to Compile");
CSharpCompilation compilation = CSharpCompilation.Create(
assemblyName,
syntaxTrees: new[] { syntaxTree },
references: references,
options: new CSharpCompilationOptions(
OutputKind.DynamicallyLinkedLibrary,
optimizationLevel: OptimizationLevel.Release));
using (var ms = new MemoryStream())
{
EmitResult result = compilation.Emit(ms);
if (!result.Success)
{
Console.WriteLine($"Compile Failed , see pod logs for more details");
IEnumerable<Diagnostic> failures = result.Diagnostics.Where(diagnostic =>
diagnostic.IsWarningAsError ||
diagnostic.Severity == DiagnosticSeverity.Error).ToList();
foreach (Diagnostic diagnostic in failures)
{
errors.Add($"{diagnostic.Id}: {diagnostic.GetMessage()}");
Console.WriteLine($"COMPILE ERROR :{diagnostic.Id}: {diagnostic.GetMessage()}", "ERROR");
}
}
else
{
oinfo.Add("COMPILE SUCCESS!!");
Console.WriteLine($"COMPILE SUCCESS!!");
ms.Seek(0, SeekOrigin.Begin);
Assembly assembly = AssemblyLoadContext.Default.LoadFromStream(ms);
//var type = assembly.GetType("FissionFunction");
//support for Namespace , as well as backward compatibility for existing functions
var type = assembly.GetTypes().FirstOrDefault(x => x.Name.EndsWith("FissionFunction"));
//assembly.GetTypes().Where(x=>x.Name.ToLower().EndsWith("FissionFunction".ToLower())).FirstOrDefault();
var info = type.GetMember("Execute").First() as MethodInfo;
return new Function(assembly, type, info);
}
}
return null;
#endregion
}
private Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgs args)
{
//This handler is called only when the common language runtime tries to bind to the assembly and fails.
Console.WriteLine($"Dynamically trying to load dll {(args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()} in parent assembaly");
//Retrieve the list of referenced assemblies in an array of AssemblyName.
Assembly MyAssembly = null, objExecutingAssemblies;
string strTempAssmbPath_relative = "", strTempAssmbPath_absolute = "";
objExecutingAssemblies = Assembly.GetExecutingAssembly();
AssemblyName[] arrReferencedAssmbNames = objExecutingAssemblies.GetReferencedAssemblies();
////Loop through the array of referenced assembly names.
//load all available dlls from deployment folder in dllinfo object
if (functionSpecification.libraries.Any(x => x.name.ToLower() == (args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()))
{
strTempAssmbPath_relative = functionSpecification.libraries.Where(x => x.name.ToLower() == (args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()).FirstOrDefault().path;
strTempAssmbPath_absolute = Path.Combine(packagepath, strTempAssmbPath_relative);
Console.WriteLine($"loading dll in parent assembaly :{strTempAssmbPath_absolute.GetrelevantPathAsPerOS()}");
//Load the assembly from the specified path.
MyAssembly = Assembly.LoadFile(strTempAssmbPath_absolute.GetrelevantPathAsPerOS());
Console.WriteLine($"Load success for :{strTempAssmbPath_absolute.GetrelevantPathAsPerOS()}");
}
if (MyAssembly == null)
{
Console.WriteLine($"WARNING !!! unabel to locate dll :{(args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()} ", "WARNING");
}
//Return the loaded assembly.
return MyAssembly;
}
}
}
environments/dotnet20/FissionContext.cs
+17
View File
@@ -4,11 +4,14 @@ using System.IO;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using Nancy;
using Newtonsoft.Json;
namespace Fission.DotNetCore.Api
{
public class FissionContext
{
public string PackagePath { get; set; }
public FissionContext(Dictionary<string, object> args, Logger logger, FissionHttpRequest request)
{
if (args == null) throw new ArgumentNullException(nameof(args));
@@ -31,6 +34,20 @@ namespace Fission.DotNetCore.Api
logger,
new FissionHttpRequest(request));
}
//this is to support aditional setting file read from source/deployment package via function
public T GetSettings<T>(string relativePath)
{
var filePath = Path.Combine(this.PackagePath, relativePath);
Console.WriteLine($"Going to Get Setting from :{filePath}");
string json = GetSettingsJson(filePath);
return JsonConvert.DeserializeObject<T>(json);
}
private string GetSettingsJson(string relativePath)
{
return File.ReadAllText(Path.Combine(this.PackagePath, relativePath));
}
}
public class Logger
environments/dotnet20/Model/BuilderRequest.cs
+27
View File
@@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Fission.DotNetCore.Model
{
public class BuilderRequest
{
/// <summary>
/// this is folder path of deployment package which contains all deployment content copied to env
/// </summary>
public string filepath { get; set; }
public string functionName { get; set; }
public string url { get; set; }
public FunctionMetadata FunctionMetadata { get; set; }
}
public class FunctionMetadata
{
public string name { get; set; }
public string @namespace { get; set; }
public string selfLink { get; set; }
public string uid { get; set; }
public string resourceVersion { get; set; }
public int generation { get; set; }
public DateTime creationTimestamp { get; set; }
}
}
environments/dotnet20/Model/DllInfo.cs
+16
View File
@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Fission.DotNetCore.Model
{
public class DllInfo
{
public string name { get; set; }
public string rootPackage { get; set; }
public string framework { get; set; }
public string processor { get; set; }
public string path { get; set; }
}
}
environments/dotnet20/Model/EnvironmentSettings.cs
+16
View File
@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Fission.DotNetCore.Model
{
public class EnvironmentSettings
{
public string LogDirectory { get; set; }
public string DllDirectory { get; set; }
public string functionBodyFileName { get; set; }
public string functionSpecFileName { get; set; }
public bool RunningOnwindows { get; set; }
}
}
environments/dotnet20/Model/FunctionSpecification.cs
+37
View File
@@ -0,0 +1,37 @@
using Fission.DotNetCore.Model;
using System;
using System.Collections.Generic;
using System.Text;
namespace Fission.DotNetCore.Model
{
public class FunctionSpecification
{
public FunctionSpecification()
{
this.libraries = new List<Library>();
}
public string functionName { get; set; }
public List<Library> libraries { get; set; }
public string hash { get; set; }
public string certificatePath { get; set; }
}
public class Library
{
public Library()
{
}
public Library(DllInfo dllInfo)
{
this.name = dllInfo.name;
this.nugetPackage = dllInfo.rootPackage;
this.path = dllInfo.path;
}
public string name { get; set; }
//public string version { get; set; }
public string path { get; set; }
public string nugetPackage { get; set; }
}
}
environments/dotnet20/README.md
+94
View File
@@ -268,3 +268,97 @@ $ curl -XPOST http://localhost:8888/specialize
```
$ curl -XGET http://localhost:8888
```
## Few Aditional Features
**1. NameSpace support :**
Now , You can use namespace for Fission function class and have many other classes in same namespace , this is also backward compatible , however main execution class would always be *FissionFunction* and its method *Execute*
```
using System;
using Fission.DotNetCore.Api;
public class FissionFunction
{
public string Execute(FissionContext context){
//orignal logic
}
public string AnotherClass(string myval){
//do something
}
}
```
**2. Aditional **setting/configuration file** support :**
Now , with Fission V2 end point with builder , in source package you can have aditional setting
files which can be read by fission function .
Lets say you are writing a function and you need some configurable option and setting to be available in function and thus you want to use some additional configuration file , then you can also achieve the same by having a JSON based configuration file and a corresponding POCO Class for the same.
Please use https://csharp2json.io/ & http://json2csharp.com/ to create correct POCO class for you JSON configuration file.
Here is an example of a such file which we want to use in function , lets say your package.zip contains :
```
Source Package zip :
--soruce.zip
|--Func.cs
|--nuget.txt
|--exclude.txt
|--mysetting.json
|--....MiscFiles(optional)
|--....MiscFiles(optional)
```
here is what ***mysetting.json*** looks like :
```
{
"name": "Alpha",
"sendGridEndPoints":
[
{ "port": 1002 },
{ "port": 3004 }
]
}
```
here is what ***func.cs*** looks like :
```
using System;
using Fission.DotNetCore.Api;
namespace FuncNameSpace
{
public class FissionFunction
{
public string Execute(FissionContext context){
string respo="initial value";
context.Logger.WriteInfo("Staring..... ");
var settings =context.GetSettings<SendGridSettings>("mysetting.json");
context.Logger.WriteInfo($"SendGridEndPoint port : {settings.SendGridEndPoints[0].port} ..... ");
respo=settings.SendGridEndPoints[0].port;
context.Logger.WriteInfo("Done!!");
return respo;
}
}
public class SendGridSettings
{
public string name { get; set; }
public System.Collections.Generic.List<SendGridEndPoint> SendGridEndPoints { get; set; }
}
public class SendGridEndPoint
{
public string port { get; set; }
}
}
```
**3. Nuget support :**
with use of fission builder we can now add various compatible nugets with our deployment package so that it can be leverage via our function code. Please go through detailed documentation of [fission builder for dotnet 2.0 environment](https://github.com/fission/fission/tree/master/environments/dotnet20/builder).
environments/dotnet20/Utility/EnvironmentExtension.cs
+33
View File
@@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Fission.DotNetCore.Utilty
{
public static class EnvironmentExtension
{
public static IEnumerable<TSource> DistinctBy<TSource, TKey>(
this IEnumerable<TSource> source,
Func<TSource, TKey> keySelector)
{
var knownKeys = new HashSet<TKey>();
return source.Where(element => knownKeys.Add(keySelector(element)));
}
public static string GetrelevantPathAsPerOS(this string curruntPath)
{
if (EnvironmentHelper.Instance.environmentSettings.RunningOnwindows && curruntPath.Contains("\\"))
{
return curruntPath;
}
else
{
return curruntPath.Replace("\\", "/");
}
}
}
}
environments/dotnet20/Utility/EnvironmentHelper.cs
+117
View File
@@ -0,0 +1,117 @@
using Fission.DotNetCore.Api;
using Fission.DotNetCore.Model;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
namespace Fission.DotNetCore.Utilty
{
public sealed class EnvironmentHelper
{
private static readonly Lazy<EnvironmentHelper> lazy =
new Lazy<EnvironmentHelper>(() => new EnvironmentHelper());
public static EnvironmentHelper Instance { get { return lazy.Value; } }
public EnvironmentSettings environmentSettings
{
get
{
if (_envSettings == null)
{
string watcherSettingsjson = GetEnvironmentSettingsJson();
_envSettings = ObjectConverter.Instance.GetWatcherSettingsFromJson(watcherSettingsjson);
}
return _envSettings;
}
set
{
_envSettings = value;
}
}
private EnvironmentSettings _envSettings;
static EnvironmentHelper()
{
}
private EnvironmentHelper()
{
}
public void writeToFile(string msg)
{
using (StreamWriter sw = new StreamWriter("main.log",true))
{
sw.AutoFlush = true;
sw.WriteLine($"{DateTime.Now} : {msg}");
}
}
private string GetEnvironmentSettingsJson()
{
var baselocation = AppDomain.CurrentDomain.BaseDirectory;
var FileLocation = baselocation + "envsettings.json";
return File.ReadAllText(FileLocation);
}
public BuilderRequest GetBuilderRequest(string json)
{
BuilderRequest builderRequest = new BuilderRequest();
try
{
builderRequest = JsonConvert.DeserializeObject<BuilderRequest>(json);
}
catch(Exception ex)
{
Console.WriteLine("Error : Unable to intersept request json " + ex.Message + ex.StackTrace);
}
return builderRequest;
}
public List<DllInfo> GetDllInfoFromDirectory(string directorypath)
{
List<DllInfo> dllInfos = new List<DllInfo>();
Console.WriteLine($"finding dll in folder {directorypath}");
DirectoryInfo d = new DirectoryInfo(directorypath);//Assuming packagepath is your Folder
var files = d.GetFiles("*.dll"); //Getting dll files
foreach(var file in files)
{
Console.WriteLine($"found dll {file.Name.ToLower()} at {Path.Combine(directorypath, file.Name)}");
dllInfos.Add(new DllInfo() {
name = file.Name.ToLower(),
path = Path.Combine(directorypath, file.Name)
});
}
return dllInfos;
}
public FunctionSpecification GetFunctionSpecs(string directorypath)
{
string functionSpecsFilePath = Path.Combine(directorypath, this.environmentSettings.functionSpecFileName);
if (File.Exists(functionSpecsFilePath))
{
string specsJson = File.ReadAllText(functionSpecsFilePath);
return ObjectConverter.Instance.GetFunctionSpecificationFromJson(specsJson);
}
else
throw new Exception($"Function Specification file not found at {functionSpecsFilePath}");
}
}
}
environments/dotnet20/Utility/ObjectConverter.cs
+36
View File
@@ -0,0 +1,36 @@
using Fission.DotNetCore.Model;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Fission.DotNetCore.Utilty
{
public sealed class ObjectConverter
{
private static readonly Lazy<ObjectConverter> lazy =
new Lazy<ObjectConverter>(() => new ObjectConverter());
public static ObjectConverter Instance { get { return lazy.Value; } }
private ObjectConverter()
{
}
public EnvironmentSettings GetWatcherSettingsFromJson(string json)
{
return JsonConvert.DeserializeObject<EnvironmentSettings>(json);
}
public FunctionSpecification GetFunctionSpecificationFromJson(string json)
{
return JsonConvert.DeserializeObject<FunctionSpecification>(json);
}
}
}
environments/dotnet20/builder/Builder.cs
+55
View File
@@ -0,0 +1,55 @@
using Builder.Utility;
using NugetWorker;
using System;
using System.IO;
using Builder.Engine;
namespace Builder
{
class Builder
{
static void Main(string[] args)
{
Console.WriteLine("Builder Task Begins!");
//create log file name for this session
var logFileName = $"{DateTime.Now.ToString("yyyy_MM_dd")}_{Guid.NewGuid().ToString()}.log";
Console.WriteLine($"going to create logger!!");
try
{
string _logdirectory = BuilderHelper.Instance.builderSettings.BuildLogDirectory;
BuilderHelper.Instance._logFileName = Path.Combine(_logdirectory, logFileName);
BuilderHelper.Instance.logger = new Utility.Logger(
BuilderHelper.Instance._logFileName);
//set the same for nuget engine dll
NugetHelper.Instance.logger = new NugetWorker.Logger(BuilderHelper.Instance._logFileName);
Console.WriteLine($"detailed logs for this build will be at : {Path.Combine(_logdirectory, logFileName)}!!");
BuilderEngine builderEngine = new BuilderEngine();
builderEngine.BuildPackage().Wait();
}
catch (Exception ex)
{
string detailedException = string.Empty;
try
{
detailedException= BuilderHelper.Instance.DeepException(ex);
Console.WriteLine($"Exception During Build : {Environment.NewLine} {ex.Message} | {ex.StackTrace} | {Environment.NewLine} {detailedException}");
}
catch(Exception childEx)
{
//do nothing , just log orignal exception
Console.WriteLine($"{Environment.NewLine} Exception During Build :{ex.Message} |{Environment.NewLine} {ex.StackTrace} {Environment.NewLine} ");
}
//now throw back exception so that build gets failed via builder
throw;
}
Console.WriteLine("Builder Task Ends!");
}
}
}
environments/dotnet20/builder/Builder.csproj
+40
View File
@@ -0,0 +1,40 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Nancy" Version="2.0.0-clinteastwood" />
<PackageReference Include="nugetdownloader" Version="2.0.0" />
<PackageReference Include="System.Runtime.Loader" Version="4.3.0" />
<PackageReference Include="System.Runtime.Serialization.Json" Version="4.3.0" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="2.9.0" />
</ItemGroup>
<ItemGroup>
<Reference Include="System">
<HintPath>System</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<None Update="build.sh">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="builder">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="builderSettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="log4net.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="nugetSettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
environments/dotnet20/builder/Builder.sln
+25
View File
@@ -0,0 +1,25 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.27906.1
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Builder", "Builder.csproj", "{D479FF57-44A0-483D-B5B0-B6C475D12292}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D479FF57-44A0-483D-B5B0-B6C475D12292}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D479FF57-44A0-483D-B5B0-B6C475D12292}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D479FF57-44A0-483D-B5B0-B6C475D12292}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D479FF57-44A0-483D-B5B0-B6C475D12292}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {AA20D0B8-E5D5-4BA1-9E29-FC5132A4DD00}
EndGlobalSection
EndGlobal
environments/dotnet20/builder/BuilderEngine/BuilderEngine.cs
+306
View File
@@ -0,0 +1,306 @@
using Builder.Model;
using Builder.Utility;
using NugetWorker;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using System.Linq;
using NuGet.Packaging;
using System.IO;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using System.Reflection;
using Microsoft.CodeAnalysis.Emit;
using System.Runtime.Loader;
using Newtonsoft.Json;
namespace Builder.Engine
{
public class BuilderEngine
{
public string SRC_PKG = string.Empty;
public string DEPLOY_PKG = string.Empty;
List<DllInfo> dllInfos = new List<DllInfo>();
List<ExcludeDll> excludeDlls = new List<ExcludeDll>();
List<IncludeNuget> includeNugets = new List<IncludeNuget>();
List<string> compile_errors = new List<string>();
List<string> compile_info = new List<string>();
public BuilderEngine()
{
SRC_PKG = Environment.GetEnvironmentVariable("SRC_PKG");
DEPLOY_PKG = Environment.GetEnvironmentVariable("DEPLOY_PKG");
}
public async Task BuildPackage()
{
await BuildDllInfo();
Console.WriteLine("DLL Info Gathered!!");
// try to compile the function and if compilation succedd ,then create func spec file
//this enables us to find compilation issues during package creation itself thus saving time
// however this feature impose that the function file name should be func.cs
//if we dont want it , we can comment the TryCompile() logic
Console.WriteLine("Trying to compile it during build itslef !!");
bool compiled =await TryCompile();
Console.WriteLine($"Compilation result Gathered as : {compiled}!!");
if (compiled)
{
//nowwhatever has been done so far and all files which are generated are in /app folder where dll resides
//thus copy relavant thing in SRC_PKG as it is ,but lest skip it here , we shall do it in build.sh
CopyToSourceDir();
Console.WriteLine($"Copy to Source Done!!");
//build the function specs
await BuildSpecs();
Console.WriteLine("BuildSpecs Done!!");
}
else
{
Console.WriteLine("Compilation failed , throwing exception !!");
foreach(var error in compile_errors)
{
Console.WriteLine($"COMPILATION ERROR : {error}");
}
throw new Exception($"COMPILATION FAILED !! , See builder logs for details , total Errors : {compile_errors.Count}");
}
}
public void CopyToSourceDir()
{
//to create folder if it doesnt already exists
string destinationFile=Path.Combine(SRC_PKG, BuilderHelper.Instance.builderSettings.DllDirectory, "dummy.txt");
new FileInfo(destinationFile).Directory.Create();
//copy all dlls
foreach (var dllinfo in dllInfos)
{
string filename = Path.GetFileName(dllinfo.path);
destinationFile = Path.Combine(SRC_PKG, BuilderHelper.Instance.builderSettings.DllDirectory, filename);
File.Copy(dllinfo.path, destinationFile,true);
}
//copy logs , well there is not point as logs are still being generated
//create dir if not exist
//new FileInfo(Path.Combine(SRC_PKG, BuilderHelper.Instance._logFileName)).Directory.Create();
//File.Copy(BuilderHelper.Instance._logFileName, Path.Combine(SRC_PKG, BuilderHelper.Instance._logFileName));
//BuilderHelper.Instance.logger.Log($"All Required Files copied to {SRC_PKG}");
}
public async Task<bool> TryCompile()
{
bool issuccess = false;
string CODE_PATH = Path.Combine(SRC_PKG, BuilderHelper.Instance.builderSettings.functionBodyFileName);
if (!File.Exists(CODE_PATH))
{
Console.WriteLine($"Source Code not found at : {CODE_PATH} !" +
$" to use TryCompile() in Builder, make sure , your main function file name is " +
$"{BuilderHelper.Instance.builderSettings.functionBodyFileName} and " +
$"it is located at root of zip!!" );
return issuccess;
}
var code = File.ReadAllText(CODE_PATH);
issuccess = await Compile(code);
return issuccess;
}
public async Task<bool> Compile(string code)
{
bool issuccess = false;
#region assymbaly init and parent dll refrences
SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(code);
string assemblyName = Path.GetRandomFileName();
var coreDir = Directory.GetParent(typeof(Enumerable).GetTypeInfo().Assembly.Location);
List<MetadataReference> references = new List<MetadataReference>
{
MetadataReference.CreateFromFile(coreDir.FullName + Path.DirectorySeparatorChar + "mscorlib.dll"),
MetadataReference.CreateFromFile(coreDir.FullName + Path.DirectorySeparatorChar + "netstandard.dll"),
MetadataReference.CreateFromFile(typeof(object).GetTypeInfo().Assembly.Location),
MetadataReference.CreateFromFile(Assembly.GetEntryAssembly().Location),
MetadataReference.CreateFromFile(typeof(System.Runtime.Serialization.Json.DataContractJsonSerializer).GetTypeInfo().Assembly.Location)
};
foreach (var referencedAssembly in Assembly.GetEntryAssembly().GetReferencedAssemblies())
{
var assembly = Assembly.Load(referencedAssembly);
references.Add(MetadataReference.CreateFromFile(assembly.Location));
BuilderHelper.Instance.logger.Log($"Refering assembaly based dls : {assembly.Location}");
}
#endregion
#region handler registration for runtime resolution
//now add handeler for missing dlls for parent app domain as same assembalies should be needed
//for parent , thus refering from https://support.microsoft.com/en-in/help/837908/how-to-load-an-assembly-at-runtime-that-is-located-in-a-folder-that-is
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.AssemblyResolve += CurrentDomain_AssemblyResolve;
#endregion
BuilderHelper.Instance.logger.Log($"dynamic handlar registered!!");
#region nuget dll refrence add
//now add those dll refrence
foreach (var dll in dllInfos)
{
BuilderHelper.Instance.logger.Log($"refering nuget based dll : {dll.path}");
references.Add(MetadataReference.CreateFromFile(dll.path));
}
#endregion
#region compilation
CSharpCompilation compilation = CSharpCompilation.Create(
assemblyName,
syntaxTrees: new[] { syntaxTree },
references: references,
options: new CSharpCompilationOptions(
OutputKind.DynamicallyLinkedLibrary,
optimizationLevel: OptimizationLevel.Release));
using (var ms = new MemoryStream())
{
EmitResult result = compilation.Emit(ms);
if (!result.Success)
{
BuilderHelper.Instance.logger.Log($"Compile Failed!!!!",true);
IEnumerable<Diagnostic> failures = result.Diagnostics.Where(diagnostic =>
diagnostic.IsWarningAsError ||
diagnostic.Severity == DiagnosticSeverity.Error).ToList();
foreach (Diagnostic diagnostic in failures)
{
compile_errors.Add($"{diagnostic.Id}: {diagnostic.GetMessage()}");
BuilderHelper.Instance.logger.Log($"COMPILE ERROR :{diagnostic.Id}: {diagnostic.GetMessage()}");
}
}
else
{
BuilderHelper.Instance.logger.Log("Compile Success!!",true);
issuccess = true;
}
}
#endregion
return issuccess;
}
public async Task BuildSpecs()
{
//create function specs C# object
FunctionSpecification functionSpecification = new FunctionSpecification();
functionSpecification.functionName = BuilderHelper.Instance.builderSettings.functionBodyFileName;
foreach (var dllinfo in dllInfos)
{
//here is the tweak , as this path is based on execution directoy , thus choose the relative path
string destinationFile = Path.Combine(BuilderHelper.Instance.builderSettings.DllDirectory, Path.GetFileName(dllinfo.path)).GetrelevantPathAsPerOS();
Library library = new Library()
{
name=dllinfo.name,
nugetPackage=dllinfo.rootPackage,
path = destinationFile
};
functionSpecification.libraries.Add(library);
}
//serilize that object to save it in json file
string funcMetaJson= JsonConvert.SerializeObject(functionSpecification);
string funcMetaFile = Path.Combine(this.SRC_PKG, BuilderHelper.Instance.builderSettings.functionSpecFileName);
BuilderHelper.Instance.WriteTofile(funcMetaFile, funcMetaJson);
}
public async Task BuildDllInfo()
{
//read the nuget file and download nuget packages
includeNugets = BuilderHelper.Instance.GetNugettoInclude(SRC_PKG);
//set the nuget logger to same logger
foreach (var nuget in includeNugets)
{
NugetEngine nugetEngine = new NugetEngine();
await nugetEngine.GetPackage(nuget.packageName, nuget.version);
//add the list of dlls received via this package in master list
dllInfos.AddRange(nugetEngine.dllInfos);
}
//now do a distinct of all dlls paths as multiple packaed might have added same dll
dllInfos = dllInfos.DistinctBy(x => x.path).ToList();
#if DEBUG
dllInfos.LogDllPathstoCSV("preFilter.CSV");
#endif
//exclude the dlls from exclude file
excludeDlls = BuilderHelper.Instance.GetDllstoExclude(SRC_PKG);
foreach (var excludedll in excludeDlls)
{
BuilderHelper.Instance.logger.Log($"trying to remove , if available : {excludedll.dllName} from package {excludedll.packageName}");
dllInfos.RemoveAll(x => x.rootPackage.ToLower() == excludedll.packageName.ToLower() && x.name.ToLower() == excludedll.dllName.ToLower());
}
#if DEBUG
//log dlls in debug mode
dllInfos.LogDllPathstoCSV("PostFilter.CSV");
#endif
}
private Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgs args)
{
//This handler is called only when the common language runtime tries to bind to the assembly and fails.
BuilderHelper.Instance.logger.Log($"Dynamically trying to load dll {(args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()} in parent assembaly");
//Retrieve the list of referenced assemblies in an array of AssemblyName.
Assembly MyAssembly = null, objExecutingAssemblies;
string strTempAssmbPath = "";
objExecutingAssemblies = Assembly.GetExecutingAssembly();
AssemblyName[] arrReferencedAssmbNames = objExecutingAssemblies.GetReferencedAssemblies();
////Loop through the array of referenced assembly names.
if (dllInfos.Any(x => x.name.ToLower() == (args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()))
{
strTempAssmbPath = dllInfos.Where(x => x.name.ToLower() == (args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()).FirstOrDefault().path;
BuilderHelper.Instance.logger.Log($"loading dll in parent :{strTempAssmbPath}");
//Load the assembly from the specified path.
MyAssembly = Assembly.LoadFile(strTempAssmbPath);
}
if (MyAssembly == null)
{
BuilderHelper.Instance.logger.Log($"WARNING !!! unabel to locate dll :{(args.Name.Substring(0, args.Name.IndexOf(",")).ToString() + ".dll").ToLower()} ", true);
}
//Return the loaded assembly.
return MyAssembly;
}
}
}
environments/dotnet20/builder/Dockerfile
+37
View File
@@ -0,0 +1,37 @@
ARG BUILDER_IMAGE=fission/builder
FROM ${BUILDER_IMAGE} AS fission-builder
FROM microsoft/dotnet:2.0.0-sdk AS builderimage
WORKDIR /app
# Copy csproj and restore as distinct layers
COPY *.csproj ./
RUN dotnet restore
# Copy everything else and build
COPY . ./
RUN dotnet publish -c Release -o out
# Build runtime image
FROM microsoft/dotnet:aspnetcore-runtime
WORKDIR /app
COPY --from=builderimage /app/out .
#this builder is actually compilation from : https://github.com/fission/fission/tree/master/builder/cmd and renamed cmd.exe to builder
# make sure to compile it in linux only else you will get exec execute error as binary was compiled in windows and running on linux
COPY --from=fission-builder /builder /builder
#ADD builder /builder
ADD build.sh /usr/local/bin/build
RUN chmod +x /usr/local/bin/build
ADD build.sh /bin/build
RUN chmod +x /bin/build
EXPOSE 8001
environments/dotnet20/builder/Model/BuilderSettings.cs
+19
View File
@@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Builder.Model
{
public class BuilderSettings
{
public string NugetSpecsFile { get; set; }
public string DllExcludeFile { get; set; }
public string BuildLogDirectory { get; set; }
public string NugetPackageRegEx { get; set; }
public string ExcludeDllRegEx { get; set; }
public bool RunningOnwindows { get; set; }
public string functionBodyFileName { get; set; }
public string functionSpecFileName { get; set; }
public string DllDirectory { get; set; }
}
}
environments/dotnet20/builder/Model/ExcludeDll.cs
+12
View File
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Builder.Model
{
public class ExcludeDll
{
public string dllName { get; set; }
public string packageName { get; set; }
}
}
environments/dotnet20/builder/Model/FissionContext.cs
+137
View File
@@ -0,0 +1,137 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using Nancy;
using Newtonsoft.Json;
namespace Fission.DotNetCore.Api
{
public class FissionContext
{
public FissionContext(Dictionary<string, object> args, Logger logger, FissionHttpRequest request)
{
if (args == null) throw new ArgumentNullException(nameof(args));
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (request == null) throw new ArgumentNullException(nameof(request));
Arguments = args;
Logger = logger;
Request = request;
}
public Dictionary<string, object> Arguments { get; private set; }
public FissionHttpRequest Request { get; private set; }
public Logger Logger { get; private set; }
public static FissionContext Build(Request request, Logger logger)
{
return new FissionContext(((DynamicDictionary)request.Query).ToDictionary(),
logger,
new FissionHttpRequest(request));
}
//this are curruntly dummy , not being implemented, just to pass compilation
//actual execution is written in environment to use the app settings as there we need it
public T GetSettings<T>(string relativePath)
{
//intentionaly doing it as these are just dummy methods not being called
//but if tomorrow if we decide to give implementation for execution in build then we
//need to implement it
throw new NotImplementedException();
}
//this are curruntly dummy , not being implemented, just to pass compilation
//actual execution is written in environment to use the app settings as there we need it
private string GetSettingsJson(string relativePath)
{
//intentionaly doing it as these are just dummy methods not being called
//but if tomorrow if we decide to give implementation for execution in build then we
//need to implement it
throw new NotImplementedException();
}
}
public class Logger
{
public void Write(Severity severity, string format, params object[] args)
{
Console.WriteLine($"{DateTime.Now.ToString("MM/dd/yy H:mm:ss zzz")} {severity}: " + format, args);
}
public void WriteInfo(string format, params object[] args)
{
Write(Severity.Info, format, args);
}
public void WriteWarning(string format, params object[] args)
{
Write(Severity.Warning, format, args);
}
public void WriteError(string format, params object[] args)
{
Write(Severity.Error, format, args);
}
public void WriteCritical(string format, params object[] args)
{
Write(Severity.Critical, format, args);
}
public void WriteVerbose(string format, params object[] args)
{
Write(Severity.Verbose, format, args);
}
}
public enum Severity
{
Info,
Warning,
Error,
Critical,
Verbose
}
public class FissionHttpRequest
{
private readonly Request _request;
internal FissionHttpRequest(Request request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
_request = request;
}
public Stream Body { get { return _request.Body; } }
public string BodyAsString()
{
int length = (int)_request.Body.Length;
byte[] data = new byte[length];
_request.Body.Read(data, 0, length);
return Encoding.UTF8.GetString(data);
}
public Dictionary<string, IEnumerable<string>> Headers
{
get
{
var headers = new Dictionary<string, IEnumerable<string>>();
foreach (var kv in _request.Headers)
{
headers.Add(kv.Key, kv.Value);
}
return headers;
}
}
public X509Certificate Certificate { get { return _request.ClientCertificate; } }
public string Url { get { return _request.Url.ToString(); } }
public string Method { get { return _request.Method; } }
}
}
environments/dotnet20/builder/Model/FunctionSpecification.cs
+40
View File
@@ -0,0 +1,40 @@
using NugetWorker;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace Builder.Model
{
public class FunctionSpecification
{
public FunctionSpecification()
{
this.libraries = new List<Library>();
}
public string functionName { get; set; }
public List<Library> libraries { get; set; }
public string hash { get; set; }
public string certificatePath { get; set; }
}
public class Library
{
public Library()
{
}
public Library(DllInfo dllInfo)
{
this.name = dllInfo.name;
this.nugetPackage = dllInfo.rootPackage;
this.path = dllInfo.path;
}
public string name { get; set; }
//public string version { get; set; }
public string path { get; set; }
public string nugetPackage { get; set; }
}
}
environments/dotnet20/builder/Model/IncludeNuget.cs
+12
View File
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Builder.Model
{
public class IncludeNuget
{
public string packageName { get; set; }
public string version { get; set; }
}
}
environments/dotnet20/builder/README.md
+175
View File
@@ -0,0 +1,175 @@
# Fission: dotnet 2.0 C# Environment Builder
This is a simple dotnet core 2.0 C# environment builder for Fission.
It's a docker image containing the dotnet 2.0.0 (core) run-time builder. This image read the source package and uses
*roslyn* to compile the source package code and creates deployment package out of it.
This enables using nuget packages as part of function and thus user can use extended functionality in fission functions via nuget.
During build , builder also does a pre-compile to prevent any compilation issues during function environment pod specialization.
Thus we get the function compilation issues during builder phase in package info's build logs itself.
**Note** : In future we can further enhance the compiled assembly to be saved as physical file in deployment package ,
as this will save cold start time for function.
Now , once after the build is finished, the output package (deploy archive) will be uploaded to storagesvc to store.
Then, during the specialization, the fetcher inside function pod will fetch the package from storagesvc for function loading
and will call on the **/v2/specialized** endpoint of fission environment with required parameters.
There further environment will compile it and execute the function.
Example of simplest possible class to be executed:
The source package structure in zip file :
```
Source Package zip :
--soruce.zip
|--func.cs
|--nuget.txt
|--exclude.txt
|--....MiscFiles(optional)
|--....MiscFiles(optional)
```
**func.cs** --> This contains original function body with Executing method name as : Execute
**nuget.txt**--> this file contains list of nuget packages required by your function , in this file put one line per nuget with nugetpackage name:version(optional) format, for example :
```
RestSharp
CsvHelper
Newtonsoft.json:10.2.1.0
```
this should match the following regex as mentioned in builderSetting.json
```
"NugetPackageRegEx": "\\:?\\s*(?<package>[^:\\n]*)(?:\\:)?(?<version>.*)?"
```
**exclude.txt**--> as nuget.txt will download original package and their dependent packages , thus sometime dependent packages might not be
that useful and can break compilation , thus this file contains list of dlls of specific nuget packages which doesn't need to be added during compilation if they break compilation .Put one line per nuget with dllname:nugetpackagename formate ,for example :
```
Newtonsoft.json:Newtonsoft.json.dll
```
this should match the following regex as mentions in builderSetting.json
```
"ExcludeDllRegEx": "\\:?\\s*(?<package>[^:\\n]*)(?:\\:)?(?<dll>.*)?",
```
From above , builder will create a deployment package with all dlls in a folder and one functionspecification file :
Deployement Package zip :
```
--Deploye.zip
|--func.cs
|--nuget.txt
|--exclude.txt
|--dll()
|--newtonsoft.json.dll
|--restsharp.dll
|--csvhelper.dll
|--logs()
|-->logFileName
|--func.meta.json // this is the functionspecific file
|--....MiscFiles(optional)
|--....MiscFiles(optional)
```
Here are commands and detailed example for the same .
lets say my source package zip name is *funccsv.zip* :
**Content of func.cs:**
```
using System;
using Fission.DotNetCore.Api;
public class FissionFunction
{
public string Execute(FissionContext context){
string respo="initial value";
try
{
context.Logger.WriteInfo("Staring..... ");
respo=$" sample object by getting Enum of CsvHelper nuget dll: { CsvHelper.Caches.NamedIndex.ToString()}";
}
catch(Exception ex)
{
context.Logger.WriteError(ex.Message);
respo = ex.Message;
}
context.Logger.WriteInfo("Done!");
return respo;
}
}
```
**Content of *nuget.txt***
```
CsvHelper
```
**Content of exclude.txt**
As we dont want to exclude any specific dll thus we shall leave it as empty.
Now check name of existing environments & functions as we want to create a unique environment for this dotnetcore if not already present
```
fission env list
fission fn list
```
Create Environment with builder (choose a unique which doesn't exist , here we have chosen : dotnetcorewithnuget )
also suppose the builder image name is fissiondotnet20-builder and hosted on dockerhub as fission/dotnet20-builder
```
fission environment create --name dotnetcorewithnuget --image fission/dotnet20-env --builder fission/dotnet20-builder
```
Verify fission-builder and fission-function namespace for new pods (pods name beginning with env name which we have given like *dotnetcorewithnuget-xxx-xxx*)
```
kubectl get pods -n fission-builder
kubectl get pods -n fission-function
```
Create Package from source zip using this environment name , this will output some package name created..
```
fission package create --src funccsv.zip --env dotnetcorewithnuget
```
Note down output package name lets say it is *funccsv-zip-xyz* now check its status using package info command , this will give the status
on what happened with builder and test compilation in builder.
```
fission package info --name funccsv-zip-xyz
```
#Status of package should be f*ailed / running / succeeded* .
Wait if the status is running , until it fails or succeeded. For detailed build logs, you can shell into builder pod in fission-builder namespace and verify log location mentioned in above command's result output.
**Note** : Even If the result is succeeded , please have a look at detailed build logs to see compilation success and builder job done.
Now If the result is succeeded , then go ahead and create function using this package.
*--entrypoint* flag is optional if your function body file name is func.cs (which it should be as builder need that), else put the filename (without extension )
```
fission fn create --name dotnetcsvtest --pkg funccsv-zip-xyz --env dotnetcorewithnuget --entrypoint "func"
```
Test the function execution :
```
fission fn test --name dotnetcsvtest
```
above would execute the function and will output the enum value as written in dll.
rest of the feature are same as normal fission environment.
**Benefit of using builder** :
1. Ability to use various nuget packages.
2. Ability to use many additional files and functions as part of deployment package.
3. Ability to know the compilation issue in advance via package logs , instead of environment giving compilation issue.
4. Reusability of same deployment package.
environments/dotnet20/builder/Utility/BuilderExtensions.cs
+21
View File
@@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Builder.Utility
{
public static class BuilderExtensions
{
public static string GetrelevantPathAsPerOS(this string curruntPath)
{
if (BuilderHelper.Instance.builderSettings.RunningOnwindows)
{
return curruntPath;
}
else
{
return curruntPath.Replace("\\","/");
}
}
}
}
environments/dotnet20/builder/Utility/BuilderHelper.cs
+180
View File
@@ -0,0 +1,180 @@
using Builder.Model;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
namespace Builder.Utility
{
public sealed class BuilderHelper
{
private static readonly Lazy<BuilderHelper> lazy =
new Lazy<BuilderHelper>(() => new BuilderHelper());
public string _logFileName = string.Empty;
public static BuilderHelper Instance { get { return lazy.Value; } }
private static Logger _logger = new Logger("Initial.log");
private BuilderSettings _builderSettings = null;
public BuilderSettings builderSettings
{
get
{
if (_builderSettings == null)
{
string builderSettingsjson = GetBuilderSettingsJson();
_builderSettings = ObjectConverter.Instance.GetBuilderSettingsFromJson(builderSettingsjson);
}
return _builderSettings;
}
set
{
builderSettings = value;
}
}
public Logger logger
{
get
{
return _logger;
}
set
{
_logger = value;
}
}
static BuilderHelper()
{
}
private BuilderHelper()
{
}
private string GetBuilderSettingsJson()
{
var baselocation = AppDomain.CurrentDomain.BaseDirectory;
var FileLocation = baselocation + "builderSettings.json";
return File.ReadAllText(FileLocation);
}
public List<IncludeNuget> GetNugettoInclude(string directoryPath)
{
List<IncludeNuget> includeNugets = new List<IncludeNuget>();
string includeNugetsFilePath = Path.Combine(directoryPath, this.builderSettings.NugetSpecsFile);
if (File.Exists(includeNugetsFilePath))
{
Regex _pkgName = new Regex(this.builderSettings.NugetPackageRegEx,
RegexOptions.Compiled);
string filetext = File.ReadAllText(includeNugetsFilePath);
var _pkgMatchCollection = _pkgName.Matches(filetext);
foreach (Match match in _pkgMatchCollection)
{
if(!string.IsNullOrWhiteSpace(match.Value))
{
string package = match.Groups["package"]?.Value?.Trim();
string version = match.Groups["version"]?.Value?.Trim();
this.logger.Log($"adding {package} | {version} to includeNugets collection");
includeNugets.Add(
new IncludeNuget()
{
packageName = package,
version = version
}
);
}
}
}
return includeNugets;
}
public List<ExcludeDll> GetDllstoExclude(string directoryPath)
{
List<ExcludeDll> excludeDlls = new List<ExcludeDll>();
string excludeDllsFilePath = Path.Combine(directoryPath, this.builderSettings.DllExcludeFile);
if (File.Exists(excludeDllsFilePath))
{
//xyzPackage:abc.dll
Regex _exclude = new Regex(this.builderSettings.ExcludeDllRegEx,
RegexOptions.Compiled);
string filetext = File.ReadAllText(excludeDllsFilePath);
var _excludeMatchCollection = _exclude.Matches(filetext);
foreach (Match match in _excludeMatchCollection)
{
if (!string.IsNullOrWhiteSpace(match.Value))
{
string _package = match.Groups["package"]?.Value?.Trim();
string _dllName = match.Groups["dll"]?.Value?.Trim();
this.logger.Log($"adding {_package} | {_dllName} to excludeDlls collection");
excludeDlls.Add(
new ExcludeDll()
{
packageName = _package,
dllName = _dllName
}
);
}
}
}
return excludeDlls;
}
public string DeepException(Exception ex)
{
string responce = string.Empty;
responce = " Exception : LEVEL 1: " + Environment.NewLine + ex.Message;
if (ex.InnerException != null)
{
responce = responce + Environment.NewLine + "LEVEL 2:" + Environment.NewLine + ex.InnerException.Message;
if (ex.InnerException.InnerException != null)
{
responce =responce + Environment.NewLine + "LEVEL 3:" + Environment.NewLine + ex.InnerException.InnerException.Message;
if (ex.InnerException.InnerException.InnerException != null)
{
responce = responce + Environment.NewLine + "LEVEL 4:" + Environment.NewLine + ex.InnerException.InnerException.InnerException.Message;
if (ex.InnerException.InnerException.InnerException.InnerException != null)
{
responce = responce + Environment.NewLine + "LEVEL 5:" + Environment.NewLine + ex.InnerException.InnerException.InnerException.InnerException.Message;
}
}
}
}
if(ex.StackTrace!=null)
{
responce = responce + "|| STACK :"+ ex.StackTrace;
}
return responce;
}
public void WriteTofile(string filenameWithPath , string content)
{
using (StreamWriter sw = new StreamWriter(filenameWithPath, false))
{
sw.AutoFlush = true;
sw.Write(content);
}
}
}
}
environments/dotnet20/builder/Utility/Logger.cs
+111
View File
@@ -0,0 +1,111 @@
using log4net;
using NuGet.Common;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
namespace Builder.Utility
{
public class Logger : NuGet.Common.ILogger
{
private ILog _ILog { get; set; }
public Logger(string logpath)
{
//read the log4net config and create logger instance form log4net
XmlDocument ConfigLoader = new XmlDocument();
ConfigLoader.Load(File.OpenRead("log4net.config"));
var repo = LogManager.CreateRepository(Assembly.GetEntryAssembly(),
typeof(log4net.Repository.Hierarchy.Hierarchy));
log4net.Config.XmlConfigurator.Configure(repo, ConfigLoader["log4net"]);
var appender = ((log4net.Appender.FileAppender)repo.GetAppenders().Where(x => x.Name == "RollingLogFileAppender").FirstOrDefault());
appender.File = logpath;// $"directoryPath/{DateTime.Now.ToString("yyyy_MM_dd")}_{Guid.NewGuid().ToString()}_.log";
appender.ActivateOptions();
_ILog = LogManager.GetLogger(typeof(Logger));
}
public void Log(string message,bool logToConsoleAsWell=false)
{
if(logToConsoleAsWell)
Console.WriteLine(message);
_ILog.Info(message);
}
public void Log(LogLevel level, string data)
{
//Console.WriteLine(data);
_ILog.Info(data);
}
public void Log(ILogMessage message)
{
//Console.WriteLine(message);
_ILog.Info(message.Message);
}
public Task LogAsync(LogLevel level, string data)
{
//Console.WriteLine(data);
_ILog.Info(data);
return null;
}
public Task LogAsync(ILogMessage message)
{
//Console.WriteLine(message);
_ILog.Info(message.Message);
return null;
}
public void LogDebug(string data)
{
//Console.WriteLine(data);
_ILog.Debug(data);
}
public void LogError(string data)
{
//Console.WriteLine(data);
_ILog.Error(data);
}
public void LogInformation(string data)
{
//Console.WriteLine(data);
_ILog.Info(data);
}
public void LogInformationSummary(string data)
{
//Console.WriteLine(data);
_ILog.Info(data);
}
public void LogMinimal(string data)
{
//Console.WriteLine(data);
_ILog.Info(data);
}
public void LogVerbose(string data)
{
//Console.WriteLine(data);
_ILog.Debug(data);
}
public void LogWarning(string data)
{
//Console.WriteLine(data);
_ILog.Warn(data);
}
}
}
environments/dotnet20/builder/Utility/ObjectConverter.cs
+31
View File
@@ -0,0 +1,31 @@
using Builder.Model;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Builder.Utility
{
public sealed class ObjectConverter
{
private static readonly Lazy<ObjectConverter> lazy =
new Lazy<ObjectConverter>(() => new ObjectConverter());
public static ObjectConverter Instance { get { return lazy.Value; } }
private ObjectConverter()
{
}
public BuilderSettings GetBuilderSettingsFromJson(string json)
{
return JsonConvert.DeserializeObject<BuilderSettings>(json);
}
}
}
environments/dotnet20/builder/build.sh
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
set -euxo pipefail
cd ${SRC_PKG}
#now start execution of custom logic dll in such a way that it should copy everything in ${SRC_PKG}
#first lets try putting sample file which will prove that this worked
echo src : ${SRC_PKG} ,dest: ${DEPLOY_PKG} >builderpaths.txt
#now run actual dll for custom builder logic
# please note as this need to be executed from app folder so that all dependent files are avilable
# else you will end up getting File not found error
cd /app
#now execute dll
dotnet Builder.dll
#copy entire content to deployment package
cp -r ${SRC_PKG} ${DEPLOY_PKG}
environments/dotnet20/builder/builderSettings.json
+11
View File
@@ -0,0 +1,11 @@
{
"NugetSpecsFile": "nuget.txt",
"DllExcludeFile": "exclude.txt",
"BuildLogDirectory": "logs",
"DllDirectory": "Dlls",
"NugetPackageRegEx": "\\s*(?<package>[^:\\n]*)(?:\\:)?(?<version>.*)?",
"ExcludeDllRegEx": "\\:?\\s*(?<package>[^:\\n]*)(?:\\:)?(?<dll>.*)?",
"RunningOnwindows": false,
"functionBodyFileName": "func.cs",
"functionSpecFileName": "func.meta.json"
}
environments/dotnet20/builder/log4net.config
+28
View File
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8" ?>
<log4net>
<appender name="Console" type="log4net.Appender.ConsoleAppender">
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date %-5level: %message%newline" />
</layout>
</appender>
<appender name="RollingLogFileAppender" type="log4net.Appender.RollingFileAppender">
<lockingModel type="log4net.Appender.FileAppender+MinimalLock"/>
<File value=""/>
<appendToFile value="true" />
<rollingStyle value="Size" />
<!--<datePattern value="yyyyMMdd-HHmm" />-->
<maxSizeRollBackups value="6" />
<preserveLogFileNameExtension value="true"/>
<staticLogFileName value="false" />
<maxSizeRollBackups value="100"/>
<maximumFileSize value="1MB"/>
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level : %message %newline %newline"/>
</layout>
</appender>
<root>
<level value="ALL"/>
<!--<appender-ref ref="Console" />-->
<appender-ref ref="RollingLogFileAppender"/>
</root>
</log4net>
environments/dotnet20/builder/nugetSettings.json
+37
View File
@@ -0,0 +1,37 @@
{
"NugetFolder": "Nugetdownload",
"DisableCache": false,
"CSVDirectory": "logs",
"RunningOnwindows": false,
"NugetRepositories": [
{
"Order": 1,
"IsPrivate": false,
"Name": "local",
"Source": "Nugetdownload", //in case of local , relative path to folder
"IsPasswordClearText": false,
"Username": "",
"Password": ""
},
{
"Order": 2,
"IsPrivate": false,
"Name": "NugetV3",
"Source": "https://api.nuget.org/v3/index.json",
"IsPasswordClearText": false,
"Username": "",
"Password": ""
}
//,
//{
// "Order": 3,
// "IsPrivate": true,
// "Name": "myPrivateRepository", //in case of private repo , if not using, then just remove this section No 3
// "Source": "https://URL-FOR-myPrivateRepository/api/nuget/nugetcore/COMPLETE-END-POINT",
// "IsPasswordClearText": true,
// "Username": "johndeo@myorg.com",
// "Password": "AlPH24GAMAc49fDELTA"
//}
]
}
environments/dotnet20/dotnet20.sln
+1 -1
View File
@@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26730.12
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "fission-dotnet20", "dotnet20\fission-dotnet20.csproj", "{3F044DE1-74E5-48F7-8D23-233C24AAA45C}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "fission-dotnet20", "fission-dotnet20.csproj", "{3F044DE1-74E5-48F7-8D23-233C24AAA45C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
environments/dotnet20/envsettings.json
+7
View File
@@ -0,0 +1,7 @@
{
"LogDirectory": "logs",
"RunningOnwindows": false,
"DllDirectory": "Dlls",
"functionBodyFileName": "func.cs",
"functionSpecFileName": "func.meta.json"
}
environments/dotnet20/fission-dotnet20.csproj
+18 -1
View File
@@ -1,10 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.0</TargetFramework>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Builder\**" />
<EmbeddedResource Remove="Builder\**" />
<None Remove="Builder\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel" Version="2.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Owin" Version="2.0.0" />
@@ -14,4 +21,14 @@
<PackageReference Include="System.Runtime.Serialization.Json" Version="4.3.0" />
</ItemGroup>
<ItemGroup>
<Folder Include="Model\" />
</ItemGroup>
<ItemGroup>
<None Update="envsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>