Fix benchmark config, download method and tests

dpanfilyonok · dpanfilyonok · commit 6b14acb6407c · 2020-12-06T18:47:50.000+03:00
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report-github.md b/benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report-github.md
@@ -0,0 +1,17 @@
+``` ini
+
+BenchmarkDotNet=v0.12.1, OS=Windows 7 SP1 (6.1.7601.0)
+Intel Celeron CPU N2830 2.16GHz, 1 CPU, 2 logical and 2 physical cores
+Frequency=2115908 Hz, Resolution=472.6103 ns, Timer=TSC
+.NET Core SDK=3.1.302
+  [Host] : .NET Core 3.1.8 (CoreCLR 4.700.20.41105, CoreFX 4.700.20.41903), X64 RyuJIT DEBUG
+
+IterationCount=10  
+
+```
+|   Method |          PathToGraph | Mean | Error |                           TEPS |
+|--------- |--------------------- |-----:|------:|-------------------------------:|
+| LevelBFS | Datas(...)r.mtx [32] |   NA |    NA | (&quot;227320&quot;, &quot;227320&quot;, &quot;814134&quot;) |
+
+Benchmarks with issues:
+  BfsBenchmark.LevelBFS: Job-WTZPYK(IterationCount=10) [PathToGraph=Datas(...)r.mtx [32]]
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report.csv b/benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report.csv
@@ -0,0 +1,2 @@
+Method;Job;AnalyzeLaunchVariance;EvaluateOverhead;MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Affinity;EnvironmentVariables;Jit;Platform;PowerPlanMode;Runtime;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;RetainVm;Server;Arguments;BuildConfiguration;Clock;EngineFactory;NuGetReferences;Toolchain;IsMutator;InvocationCount;IterationCount;IterationTime;LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount;PathToGraph;Mean;Error;TEPS
+LevelBFS;Job-WTZPYK;False;Default;Default;Default;Default;Default;Default;11;Empty;RyuJit;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET Core 3.1;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;1;10;Default;Default;Default;Default;Default;Default;Default;16;Default;Datas(...)r.mtx [32];NA;NA;"(""227320"", ""227320"", ""814134"")"
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report.html b/benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report.html
@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang='en'>
+<head>
+<meta charset='utf-8' />
+<title>GraphBLAS.FSharp.Benchmarks.BfsBenchmark-20201206-170542</title>
+
+<style type="text/css">
+	table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
+	td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
+	tr { background-color: #fff; border-top: 1px solid #ccc; }
+	tr:nth-child(even) { background: #f8f8f8; }
+</style>
+</head>
+<body>
+<pre><code>
+BenchmarkDotNet=v0.12.1, OS=Windows 7 SP1 (6.1.7601.0)
+Intel Celeron CPU N2830 2.16GHz, 1 CPU, 2 logical and 2 physical cores
+Frequency=2115908 Hz, Resolution=472.6103 ns, Timer=TSC
+.NET Core SDK=3.1.302
+  [Host] : .NET Core 3.1.8 (CoreCLR 4.700.20.41105, CoreFX 4.700.20.41903), X64 RyuJIT DEBUG
+</code></pre>
+<pre><code>IterationCount=10  
+</code></pre>
+
+<table>
+<thead><tr><th>Method</th><th>   PathToGraph</th><th>Mean</th><th>Error</th><th>                    TEPS</th>
+</tr>
+</thead><tbody><tr><td>LevelBFS</td><td>Datas(...)r.mtx [32]</td><td>NA</td><td>NA</td><td>(&quot;227320&quot;, &quot;227320&quot;, &quot;814134&quot;)</td>
+</tr></tbody></table>
+</body>
+</html>
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/Config.fs b/benchmarks/GraphBLAS-sharp.Benchmarks/Config.fs
@@ -6,15 +6,26 @@ open BenchmarkDotNet.Configs
 open BenchmarkDotNet.Columns
 open BenchmarkDotNet.Reports
 open BenchmarkDotNet.Running
+open System.IO
 
 type TEPSColumn() =
     interface IColumn with
         member this.AlwaysShow: bool = true
         member this.Category: ColumnCategory = ColumnCategory.Statistics
         member this.ColumnName: string = "TEPS"
         member this.GetValue(summary: Summary, benchmarkCase: BenchmarkCase): string =
-            // let a = summary.[benchmarkCase] .ResultStatistics.Mean
-            "?????????????"
+            let meanTime = summary.[benchmarkCase].ResultStatistics.Mean
+            let pathToGraph = benchmarkCase.Parameters.["PathToGraph"].ToString()
+            match Path.GetExtension pathToGraph with
+            | ".mtx" ->
+                use streamReader = new StreamReader(pathToGraph)
+                while streamReader.Peek() = int '%' do
+                    streamReader.ReadLine() |> ignore
+                let matrixInfo = streamReader.ReadLine().Split(' ')
+                let (nrows, ncols, nnz) = float matrixInfo.[0], float matrixInfo.[1], float matrixInfo.[2]
+                let (vertices, edges) = if nrows = ncols then (nrows, nnz) else (ncols, nrows)
+                sprintf "%f" (edges / meanTime)
+            | _ -> "file`s format not supported"
         member this.GetValue(summary: Summary, benchmarkCase: BenchmarkCase, style: SummaryStyle): string =
             (this :> IColumn).GetValue(summary, benchmarkCase)
         member this.Id: string = "TEPSColumn"
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/Datasets/1/graphs.csv b/benchmarks/GraphBLAS-sharp.Benchmarks/Datasets/1/graphs.csv
@@ -1,2 +1,2 @@
-GraphName,OutputFile,GraphProvider,Url
-coAuthorsCiteseer,coAuthorsCiteseer.mtx,networkrepo,http://nrvis.com/download/data/dimacs10/coAuthorsCiteseer.zip
+GraphName,ArchiveType,Url
+coAuthorsCiteseer,zip,http://nrvis.com/download/data/dimacs10/coAuthorsCiteseer.zip
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/Datasets/GAP/graphs.csv b/benchmarks/GraphBLAS-sharp.Benchmarks/Datasets/GAP/graphs.csv
@@ -1,4 +1,4 @@
-GraphName,OutputFile,GraphProvider,Url
-twitter,,,http://an.kaist.ac.kr/~haewoon/release/twitter_social_graph/twitter_rv.tar.gz
-web,,,https://sparse.tamu.edu/MM/LAW/sk-2005.tar.gz
-road,,,http://www.dis.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.USA.gr.gz
+GraphName,ArchiveType,Url
+twitter,tar.gz,http://an.kaist.ac.kr/~haewoon/release/twitter_social_graph/twitter_rv.tar.gz
+web,tar.gz,https://sparse.tamu.edu/MM/LAW/sk-2005.tar.gz
+road,gz,http://www.dis.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.USA.gr.gz
diff --git a/benchmarks/GraphBLAS-sharp.Benchmarks/DownloadGraphs.fsx b/benchmarks/GraphBLAS-sharp.Benchmarks/DownloadGraphs.fsx
@@ -1,35 +1,25 @@
 #r "nuget: FSharp.Data"
-#r "nuget: ShellProgressBar"
+#r "nuget: Luna.ConsoleProgressBar"
 
 open System
 open System.IO
 open System.Net
 open System.IO.Compression
 open FSharp.Data
 open FSharp.Data.CsvExtensions
-open ShellProgressBar
+open Luna.ConsoleProgressBar
 
-let downloadGraphs graphProvider url (outputFile: string) =
-    printfn "%s %s %s" graphProvider url (Path.GetFileName outputFile)
-    let options = ProgressBarOptions()
-    // options.ProgressCharacter <- '─'
-    // options.ProgressBarOnBottom <- true
-
-    use bar = new ProgressBar(100000, "qwe")
-    let progress = bar.AsProgress<float>()
+let downloadGraphs graphName archiveType url (outputDir: string) =
     use client = new WebClient()
-    client.DownloadProgressChanged.Add (fun e ->
-        progress.Report (float e.ProgressPercentage / 100.)
-    )
-    match graphProvider with
-    | "networkrepo" ->
-        let archive = Path.ChangeExtension(outputFile, ".zip")
-        client.AsyncDownloadFile(Uri url, archive)
-        |> Async.RunSynchronously
-        ZipFile
-            .OpenRead(archive)
-            .GetEntry(Path.GetFileName outputFile)
-            .ExtractToFile(outputFile)
+    // use bar = new ConsoleProgressBar()
+    // client.DownloadProgressChanged.Add (fun e ->
+    //     bar.Report (float e.ProgressPercentage / 100.)
+    // )
+    match archiveType with
+    | "zip" ->
+        let archive = Path.Combine [| outputDir; Path.ChangeExtension(graphName, ".zip") |]
+        client.AsyncDownloadFile(Uri url, archive) |> Async.RunSynchronously
+        ZipFile.ExtractToDirectory(archive, outputDir)
     | _ -> ()
 
 seq {
@@ -41,6 +31,5 @@ seq {
     CsvFile.Load(pathToCsv, separators=",", hasHeaders=true).Rows |> Seq.allPairs <| Seq.singleton pathToCsv)
 |> Seq.iter (fun (row, pathToCsv) ->
     let datasetRootPath = Path.GetDirectoryName pathToCsv
-    let datasetOutputPath = Path.Join [| datasetRootPath; row?OutputFile |]
-    downloadGraphs row?GraphProvider row?Url datasetOutputPath)
+    downloadGraphs row?GraphName row?ArchiveType row?Url datasetRootPath)
 
diff --git a/src/GraphBLAS-sharp/GlobalContext.fs b/src/GraphBLAS-sharp/GlobalContext.fs
@@ -0,0 +1,17 @@
+namespace GraphBLAS.FSharp
+
+open Brahma.FSharp.OpenCL.WorkflowBuilder.Evaluation
+
+[<AutoOpen>]
+module GlobalContext =
+    type MatrixBackendFormat =
+        | CSR
+        | COO
+        | Dense
+
+    type VectorBackendFormat =
+        | Sparse
+        | Dense
+
+    let mutable oclContext = OpenCLEvaluationContext()
+    let mutable matrixBackendFormat = CSR
diff --git a/src/GraphBLAS-sharp/GraphBLAS-sharp.fsproj b/src/GraphBLAS-sharp/GraphBLAS-sharp.fsproj
@@ -9,7 +9,7 @@
   <ItemGroup>
     <Compile Include="AssemblyInfo.fs" />
     <Compile Include="Helpers.fs" />
-    <Compile Include="OpenCLContext.fs" />
+    <Compile Include="GlobalContext.fs" />
     <Compile Include="Operators.fs" />
     <Compile Include="Monoid.fs" />
     <Compile Include="Semiring.fs" />
diff --git a/src/GraphBLAS-sharp/Implementations.fs b/src/GraphBLAS-sharp/Implementations.fs
@@ -3,7 +3,7 @@ namespace GraphBLAS.FSharp
 open Brahma.OpenCL
 open Brahma.FSharp.OpenCL.Core
 open Brahma.FSharp.OpenCL.Extensions
-open OpenCLContext
+open GlobalContext
 open Helpers
 open FSharp.Quotations.Evaluator
 open Brahma.FSharp.OpenCL.WorkflowBuilder.Basic
@@ -74,7 +74,7 @@ type CSRMatrix<'a when 'a : struct and 'a : equality>(csrTuples: CSRFormat<'a>)
             return! ToHost resultVector
         }
 
-        upcast DenseVector(currentContext.RunSync eval, semiring.PlusMonoid)
+        upcast DenseVector(oclContext.RunSync eval, semiring.PlusMonoid)
 
     member this.Values = csrTuples.Values
     member this.Columns = csrTuples.Columns
diff --git a/src/GraphBLAS-sharp/OpenCLContext.fs b/src/GraphBLAS-sharp/OpenCLContext.fs
diff --git a/tests/GraphBLAS-sharp.Tests/FodyWeavers.xml b/tests/GraphBLAS-sharp.Tests/FodyWeavers.xml
@@ -0,0 +1,3 @@
+﻿<Weavers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FodyWeavers.xsd">
+  <ExtraConstraints />
+</Weavers>
diff --git a/tests/GraphBLAS-sharp.Tests/FodyWeavers.xsd b/tests/GraphBLAS-sharp.Tests/FodyWeavers.xsd
@@ -0,0 +1,26 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <!-- This file was generated by Fody. Manual changes to this file will be lost when your project is rebuilt. -->
+  <xs:element name="Weavers">
+    <xs:complexType>
+      <xs:all>
+        <xs:element name="ExtraConstraints" minOccurs="0" maxOccurs="1" type="xs:anyType" />
+      </xs:all>
+      <xs:attribute name="VerifyAssembly" type="xs:boolean">
+        <xs:annotation>
+          <xs:documentation>'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed.</xs:documentation>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="VerifyIgnoreCodes" type="xs:string">
+        <xs:annotation>
+          <xs:documentation>A comma-separated list of error codes that can be safely ignored in assembly verification.</xs:documentation>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="GenerateXsd" type="xs:boolean">
+        <xs:annotation>
+          <xs:documentation>'false' to turn off automatic generation of the XML Schema file.</xs:documentation>
+        </xs:annotation>
+      </xs:attribute>
+    </xs:complexType>
+  </xs:element>
+</xs:schema>
diff --git a/tests/GraphBLAS-sharp.Tests/GraphBLAS-sharp.Tests.fsproj b/tests/GraphBLAS-sharp.Tests/GraphBLAS-sharp.Tests.fsproj
@@ -8,8 +8,8 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="AssemblyInfo.fs" />
-    <Compile Include="Tests.fs" />
+    <Compile Include="OperationsTests/VxmTests.fs" />
     <Compile Include="Main.fs" />
   </ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
-</Project>
+</Project>
diff --git a/tests/GraphBLAS-sharp.Tests/Main.fs b/tests/GraphBLAS-sharp.Tests/Main.fs
@@ -1,6 +1,8 @@
-module ExpectoTemplate
+namespace GraphBLAS.FSharp.Tests
 
 open Expecto
 
-[<EntryPoint>]
-let main argv = runTestsInAssembly defaultConfig argv
+module Main =
+
+    [<EntryPoint>]
+    let main argv = runTestsInAssembly defaultConfig argv
diff --git a/tests/GraphBLAS-sharp.Tests/OperationsTests/VxmTests.fs b/tests/GraphBLAS-sharp.Tests/OperationsTests/VxmTests.fs
@@ -0,0 +1,79 @@
+namespace GraphBLAS.FSharp.Tests
+
+open Expecto
+open System
+
+type VectorType =
+    | Sparse = 0
+    | Dense = 1
+
+type MatrixType =
+    | CSR = 0
+    | COO = 1
+    | Dense = 2
+
+type MaskType =
+    | Regular = 0
+    | Complemented = 1
+    | None = 2
+
+type OperationCase = {
+    VectorCaseType: VectorType
+    MatrixCaseType: MatrixType
+    MaskCaseType: MaskType
+}
+
+module VxmTests =
+    let rec cartesian lstlst =
+        match lstlst with
+        | [x] ->
+            List.fold (fun acc elem -> [elem]::acc) [] x
+        | h::t ->
+            List.fold (fun cacc celem ->
+                (List.fold (fun acc elem -> (elem::celem)::acc) [] h) @ cacc
+                ) [] (cartesian t)
+        | _ -> []
+
+    /// Return all values for an enumeration type
+    let enumValues (enumType: Type) : int list =
+        let values = Enum.GetValues enumType
+        let lb = values.GetLowerBound 0
+        let ub = values.GetUpperBound 0
+        [lb .. ub] |> List.map (fun i -> values.GetValue i :?> int)
+
+    let testCases =
+        [
+            typeof<VectorType>
+            typeof<MatrixType>
+            typeof<MaskType>
+        ]
+        |> List.map enumValues
+        |> cartesian
+        |> List.map (fun list -> {
+                VectorCaseType = enum<VectorType> list.[0]
+                MatrixCaseType = enum<MatrixType> list.[1]
+                MaskCaseType = enum<MaskType> list.[2]
+            })
+
+    let matrix =
+        array2D [ [ 12; 0; 7 ]
+                  [ 1; 0; 0 ]
+                  [ 3; 6; 9 ] ]
+
+    let vector = [| 1; 1; 0 |]
+
+    [<Tests>]
+    let testList =
+        testList "aaa" (
+            testCases
+            |> List.collect (fun case ->
+                    let a = 1 // определеям контекст
+                    testParam case [
+                        "2", fun value () -> Expect.equal a a "ss"
+                    ] |> List.ofSeq
+                )
+        )
+
+// member this.Baddimensions() = ()
+// let corectness = ()
+// let zerovalue = ()
diff --git a/tests/GraphBLAS-sharp.Tests/Tests.fs b/tests/GraphBLAS-sharp.Tests/Tests.fs

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Method;Job;AnalyzeLaunchVariance;EvaluateOverhead;MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Affinity;EnvironmentVariables;Jit;Platform;PowerPlanMode;Runtime;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;RetainVm;Server;Arguments;BuildConfiguration;Clock;EngineFactory;NuGetReferences;Toolchain;IsMutator;InvocationCount;IterationCount;IterationTime;LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount;PathToGraph;Mean;Error;TEPS
	`2`	`+LevelBFS;Job-WTZPYK;False;Default;Default;Default;Default;Default;Default;11;Empty;RyuJit;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET Core 3.1;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;1;10;Default;Default;Default;Default;Default;Default;Default;16;Default;Datas(...)r.mtx [32];NA;NA;"(""227320"", ""227320"", ""814134"")"`