github · hvitved · Jan 5, 2022 · michaelnebel · Jan 6, 2022 · michaelnebel
@@ -6,6 +6,7 @@
 using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Globalization;
+using Semmle.Util;
 using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CIL.Driver
@@ -169,18 +170,14 @@ public void ResolveReferences()
     /// <summary>
     /// Parses the command line and collates a list of DLLs/EXEs to extract.
     /// </summary>
-    internal class ExtractorOptions
+    internal class ExtractorOptions : CommonOptions
     {
         private readonly AssemblyList assemblyList = new AssemblyList();
 
         public ExtractorOptions(string[] args)
         {
-            Verbosity = Verbosity.Info;
-            Threads = System.Environment.ProcessorCount;
             PDB = true;
-            TrapCompression = TrapWriter.CompressionMode.Gzip;
-
-            ParseArgs(args);
+            this.ParseArguments(args);
 
             AddFrameworkDirectories(false);
 
@@ -203,12 +200,6 @@ private void AddFrameworkDirectories(bool extractAll)
             AddDirectory(RuntimeEnvironment.GetRuntimeDirectory(), extractAll);
         }
 
-        public Verbosity Verbosity { get; private set; }
-        public bool NoCache { get; private set; }
-        public int Threads { get; private set; }
-        public bool PDB { get; private set; }
-        public TrapWriter.CompressionMode TrapCompression { get; private set; }
-
         private void AddFileOrDirectory(string path)
         {
             path = Path.GetFullPath(path);
@@ -237,43 +228,25 @@ private void AddFileOrDirectory(string path)
         /// </summary>
         public IEnumerable<AssemblyName> MissingReferences => assemblyList.MissingReferences;
 
-        private void ParseArgs(string[] args)
+        public override bool HandleFlag(string flag, bool value)
         {
-            foreach (var arg in args)
+            switch (flag)
             {
-                if (arg == "--verbose")
-                {
-                    Verbosity = Verbosity.All;
-                }
-                else if (arg == "--silent")
-                {
-                    Verbosity = Verbosity.Off;
-                }
-                else if (arg.StartsWith("--verbosity:"))
-                {
-                    Verbosity = (Verbosity)int.Parse(arg.Substring(12));
-                }
-                else if (arg == "--dotnet")
-                {
-                    AddFrameworkDirectories(true);
-                }
-                else if (arg == "--nocache")
-                {
-                    NoCache = true;
-                }
-                else if (arg.StartsWith("--threads:"))
-                {
-                    Threads = int.Parse(arg.Substring(10));
-                }
-                else if (arg == "--no-pdb")
-                {
-                    PDB = false;
-                }
-                else
-                {
-                    AddFileOrDirectory(arg);
-                }
+                case "dotnet":
+                    if (value)
+                        AddFrameworkDirectories(true);
+                    return true;
+                default:
+                    return base.HandleFlag(flag, value);
             }
         }
+
+        public override bool HandleArgument(string argument)
+        {
+            AddFileOrDirectory(argument);
+            return true;
+        }
+
+        public override void InvalidArgument(string argument) { }
     }
 }
@@ -20,11 +20,11 @@ private static void DisplayHelp()
             Console.WriteLine("    path       A directory/dll/exe to analyze");
         }
 
-        private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression)
+        private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, CommonOptions options)
         {
             var sw = new Stopwatch();
             sw.Start();
-            Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _);
+            Analyser.ExtractCIL(layout, assemblyPath, logger, options, out _, out _);
             sw.Stop();
             logger.Log(Severity.Info, "  {0} ({1})", assemblyPath, sw.Elapsed);
         }
@@ -43,8 +43,7 @@ public static void Main(string[] args)
 
             var actions = options.AssembliesToExtract
                 .Select(asm => asm.Filename)
-                .Select<string, Action>(filename =>
-                    () => ExtractAssembly(layout, filename, logger, options.NoCache, options.PDB, options.TrapCompression))
+                .Select<string, Action>(filename => () => ExtractAssembly(layout, filename, logger, options))
                 .ToArray();
 
             foreach (var missingRef in options.MissingReferences)

@@ -25,22 +25,22 @@ private static void ExtractCIL(TracingExtractor extractor, TrapWriter trapWriter
         /// <param name="extractPdbs">Whether to extract PDBs.</param>
         /// <param name="trapFile">The path of the trap file.</param>
         /// <param name="extracted">Whether the file was extracted (false=cached).</param>
-        public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
+        public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
         {
             trapFile = "";
             extracted = false;
             try
             {
                 var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
                 var pathTransformer = new PathTransformer(canonicalPathCache);
-                var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer);
+                var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
                 var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
                 var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
-                using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
+                using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), options.TrapCompression, discardDuplicates: true);
                 trapFile = trapWriter.TrapFile;
-                if (nocache || !System.IO.File.Exists(trapFile))
+                if (!options.Cache || !System.IO.File.Exists(trapFile))
                 {
-                    ExtractCIL(extractor, trapWriter, extractPdbs);
+                    ExtractCIL(extractor, trapWriter, options.PDB);
                     extracted = true;
                 }
             }

@@ -25,7 +25,7 @@ public override IEnumerable<IExtractionProduct> Contents
                 else
                     Context.TrapWriter.Archive(TransformedPath, text);
 
-                yield return Tuples.file_extraction_mode(this, 2);
+                yield return Tuples.file_extraction_mode(this, Context.Extractor.Mode | ExtractorMode.Pdb);
             }
         }
     }

@@ -206,7 +206,7 @@ internal static class Tuples
         internal static Tuple files(File file, string fullName) =>
             new Tuple("files", file, fullName);
 
-        internal static Tuple file_extraction_mode(File file, int mode) =>
+        internal static Tuple file_extraction_mode(File file, ExtractorMode mode) =>
             new Tuple("file_extraction_mode", file, mode);
 
         internal static Tuple folders(Folder folder, string path) =>

@@ -64,7 +64,7 @@ private class AssemblyConstructorFactory : CachedEntityFactory<Microsoft.CodeAna
 
         public static Assembly CreateOutputAssembly(Context cx)
         {
-            if (cx.Extractor.Standalone)
+            if (cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
                 throw new InternalError("Attempting to create the output assembly in standalone extraction mode");
             return AssemblyConstructorFactory.Instance.CreateEntity(cx, outputAssemblyCacheKey, null);
         }

@@ -53,7 +53,7 @@ public override void Populate(TextWriter trapFile)
 
             if (attributeSyntax is not null)
             {
-                if (!Context.Extractor.Standalone)
+                if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
                 {
                     trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
                 }

@@ -85,7 +85,7 @@ public virtual IEnumerable<Extraction.Entities.Location> Locations
                 {
                     // Some built in operators lack locations, so loc is null.
                     yield return Context.CreateLocation(ReportingLocation);
-                    if (!Context.Extractor.Standalone && loc.Kind == LocationKind.SourceFile)
+                    if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && loc.Kind == LocationKind.SourceFile)
                         yield return Assembly.CreateOutputAssembly(Context);
                 }
             }

@@ -129,7 +129,7 @@ private static bool IsDynamicCall(ExpressionNodeInfo info)
                         .Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
                         .Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
 
-                    return Context.Extractor.Standalone ?
+                    return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
                         candidates.FirstOrDefault() :
                         candidates.SingleOrDefault();
                 }

@@ -61,7 +61,7 @@ public override void Populate(TextWriter trapFile)
                 }
             }
 
-            trapFile.file_extraction_mode(this, Context.Extractor.Standalone ? 1 : 0);
+            trapFile.file_extraction_mode(this, Context.Extractor.Mode);
         }
 
         private bool IsPossiblyTextFile()

@@ -16,7 +16,7 @@ public sealed override void Populate(TextWriter trapFile)
             trapFile.preprocessor_directive_active(this, Symbol.IsActive);
             trapFile.preprocessor_directive_location(this, Context.CreateLocation(ReportingLocation));
 
-            if (!Context.Extractor.Standalone)
+            if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
             {
                 var compilation = Compilation.Create(Context);
                 trapFile.preprocessor_directive_compilation(this, compilation);

@@ -108,7 +108,7 @@ public override IEnumerable<Extraction.Entities.Location> Locations
                 foreach (var l in GetLocations(Symbol))
                     yield return Context.CreateLocation(l);
 
-                if (!Context.Extractor.Standalone && Symbol.DeclaringSyntaxReferences.Any())
+                if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && Symbol.DeclaringSyntaxReferences.Any())
                     yield return Assembly.CreateOutputAssembly(Context);
             }
         }

@@ -178,7 +178,7 @@ private void DoExtractCIL(PortableExecutableReference r)
         {
             var stopwatch = new Stopwatch();
             stopwatch.Start();
-            CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted);
+            CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, options, out var trapFile, out var extracted);
             stopwatch.Stop();
             ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
         }

@@ -15,7 +15,7 @@ public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions
         {
             compilation = compilationIn;
             layout = new Layout();
-            extractor = new StandaloneExtractor(Logger, PathTransformer);
+            extractor = new StandaloneExtractor(Logger, PathTransformer, options);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();

@@ -49,7 +49,7 @@ public bool BeginInitialize(IEnumerable<string> roslynArgs)
             this.layout = new Layout();
             this.options = options;
             this.compilation = compilation;
-            this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer);
+            this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
             LogDiagnostics();
 
             SetReferencePaths();
@@ -188,7 +188,7 @@ private IEnumerable<Diagnostic> FilteredDiagnostics
         {
             get
             {
-                return extractor is null || extractor.Standalone || compilation is null ? Enumerable.Empty<Diagnostic>() :
+                return extractor is null || extractor.Mode.HasFlag(ExtractorMode.Standalone) || compilation is null ? Enumerable.Empty<Diagnostic>() :
                     compilation.
                     GetDiagnostics().
                     Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id));

@@ -82,7 +82,7 @@ private void ExtractTypeDeclaration(BaseTypeDeclarationSyntax node)
 
         public override void VisitAttributeList(AttributeListSyntax node)
         {
-            if (Cx.Extractor.Standalone)
+            if (Cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
                 return;
 
             var outputAssembly = Assembly.CreateOutputAssembly(Cx);

@@ -731,7 +731,7 @@ internal static void locations_mapped(this System.IO.TextWriter trapFile, NonGen
             trapFile.WriteTuple("locations_mapped", l1, l2);
         }
 
-        internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, int mode)
+        internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, ExtractorMode mode)
         {
             trapFile.WriteTuple("file_extraction_mode", file, mode);
         }

@@ -400,7 +400,7 @@ private void ExtractionError(InternalError error)
 
         private void ReportError(InternalError error)
         {
-            if (!Extractor.Standalone)
+            if (!Extractor.Mode.HasFlag(ExtractorMode.Standalone))
                 throw error;
 
             ExtractionError(error);

@@ -8,7 +8,7 @@ namespace Semmle.Extraction
     /// </summary>
     public abstract class Extractor
     {
-        public abstract bool Standalone { get; }
+        public abstract ExtractorMode Mode { get; }
 
         /// <summary>
         /// Creates a new extractor instance for one compilation unit.

@@ -0,0 +1,16 @@
+using System;
+
+namespace Semmle.Extraction
+{
+    /// <summary>
+    /// The mode in which a file is extracted.
+    /// </summary>
+    [Flags]
+    public enum ExtractorMode
+    {
+        None = 0,
+        Standalone = 1,
+        Pdb = 2,
+        QlTest = 4,
+    }
+}
@@ -4,15 +4,19 @@ namespace Semmle.Extraction
 {
     public class StandaloneExtractor : Extractor
     {
-        public override bool Standalone => true;
+        private readonly ExtractorMode mode;
+        public override ExtractorMode Mode => mode;
-        private readonly ExtractorMode mode;
-        public override ExtractorMode Mode => mode;
+        public override ExtractorMode Mode { get; };
-        private readonly ExtractorMode mode;
-        public override ExtractorMode Mode => mode;
+        public override ExtractorMode Mode { get; };
 
         /// <summary>
         /// Creates a new extractor instance for one compilation unit.
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public StandaloneExtractor(ILogger logger, PathTransformer pathTransformer) : base(logger, pathTransformer)
+        public StandaloneExtractor(ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
         {
+            mode = ExtractorMode.Standalone;
+            if (options.QlTest)
+                mode |= ExtractorMode.QlTest;
-            mode = ExtractorMode.Standalone;
-            if (options.QlTest)
-                mode |= ExtractorMode.QlTest;
+            Mode = ExtractorMode.Standalone;
+            if (options.QlTest)
+                Mode |= ExtractorMode.QlTest;
-            mode = ExtractorMode.Standalone;
-            if (options.QlTest)
-                mode |= ExtractorMode.QlTest;
+            Mode = ExtractorMode.Standalone;
+            if (options.QlTest)
+                Mode |= ExtractorMode.QlTest;
         }
     }
 }
@@ -4,8 +4,8 @@ namespace Semmle.Extraction
 {
     public class TracingExtractor : Extractor
     {
-        public override bool Standalone => false;
-
+        private readonly ExtractorMode mode;
+        public override ExtractorMode Mode => mode;
         public string OutputPath { get; }
 
         /// <summary>
@@ -14,9 +14,12 @@ public class TracingExtractor : Extractor
         /// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer) : base(logger, pathTransformer)
+        public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(logger, pathTransformer)
         {
             OutputPath = outputPath;
+            mode = ExtractorMode.None;
+            if (options.QlTest)
+                mode |= ExtractorMode.QlTest;
         }
     }
 }