From c386422a37ceef0cb4fdb73126d428f9f43f2760 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 21 May 2026 19:52:17 -0400 Subject: [PATCH 1/6] finish the old new postprocessor --- .../jlab/analysis/postprocess/Processor.java | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java index f36955c7dd..501828c1c4 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java @@ -23,6 +23,9 @@ import org.jlab.detector.scalers.DaqScalersSequence; import org.jlab.detector.helicity.HelicityBit; import org.jlab.detector.helicity.HelicitySequenceDelayed; +import org.jlab.jnp.hipo4.io.HipoWriterSorted; +import org.jlab.utils.options.OptionParser; +import org.jlab.utils.system.ClasUtilsFile; /** * @@ -56,6 +59,10 @@ public Processor(String dir, String glob, boolean restream, boolean rebuild) { configure(findPreloadFiles(dir,glob), restream, rebuild); } + public Processor(List files, boolean restream, boolean rebuild) { + configure(files, restream, rebuild); + } + public Processor(SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequence s) { conman = new ConstantsManager(); conman.init(CCDB_TABLES); @@ -301,8 +308,48 @@ private static void replace(Map files) { } } + /** + * The "postprocess" program. + * @param args + */ public static void main(String args[]) { - Processor p = new Processor(System.getenv("HOME")+"/tmp","r*.hipo",false,false); + + OptionParser o = new OptionParser("postprocess"); + o.addOption("-f","0","reflip: rebuild the HEL::flip bank"); + o.addOption("-c","0","recharge: rebuild the RUN/HEL::scaler banks"); + o.addOption("-o",null,"merged output file path"); + o.setRequiresInputList(true); + o.parse(args); + + boolean restream = !o.getOption("-f").isDefault(); + boolean rebuild = !o.getOption("-c").isDefault(); + + Processor post = new Processor(o.getInputList(), restream, rebuild); + + HipoWriterSorted writer = null; + + if (!o.getOption("-o").isDefault()) { + writer = new HipoWriterSorted(); + SchemaFactory schema = writer.getSchemaFactory(); + schema.initFromDirectory(ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4")); + writer.setCompressionType(2); + writer.open(o.getOption("-o").stringValue()); + } + + for (String f : o.getInputList()) { + HipoReader reader = new HipoReader(); + reader.setTags(0); + reader.open(f); + Event event = new Event(); + while (reader.hasNext()) { + reader.nextEvent(event); + post.processEvent(event); + if (writer != null) writer.addEvent(event); + } + reader.close(); + } + + if (writer != null) writer.close(); } } From df0b804876a3d5c924c447bad827c031df3e818d Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 22 May 2026 18:06:56 -0400 Subject: [PATCH 2/6] add new postprocessor --- bin/postprocess2 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 bin/postprocess2 diff --git a/bin/postprocess2 b/bin/postprocess2 new file mode 100755 index 0000000000..b10c9ff07a --- /dev/null +++ b/bin/postprocess2 @@ -0,0 +1,10 @@ +#!/bin/bash + +. `dirname $0`/../libexec/env.sh + +export MALLOC_ARENA_MAX=1 + +java ${JAVA_OPTS-} -Xmx768m -Xms768m -XX:+UseSerialGC \ + -cp ${COATJAVA_CLASSPATH:-''} \ + org.jlab.analysis.postprocess.Processor \ + $* From b1291a7990473b27cd9f5e8ffe025ad46b1eb18c Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 22 May 2026 18:12:46 -0400 Subject: [PATCH 3/6] remove constructor --- .../java/org/jlab/analysis/postprocess/Processor.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java index 501828c1c4..a9ac84af8e 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java @@ -63,16 +63,6 @@ public Processor(List files, boolean restream, boolean rebuild) { configure(files, restream, rebuild); } - public Processor(SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequence s) { - conman = new ConstantsManager(); - conman.init(CCDB_TABLES); - schemaFactory = schema; - helicitySequence = h; - chargeSequence = s; - runConfig = new Bank(schemaFactory.getSchema("RUN::config")); - recEvent = new Bank(schemaFactory.getSchema("REC::Event")); - } - private void configure(List preloadFiles, boolean restream, boolean rebuild) { if (!preloadFiles.isEmpty()) { HipoReader r = new HipoReader(); From 05784689addfbf08fadb795a666db22e7cc7bb5d Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 22 May 2026 18:34:02 -0400 Subject: [PATCH 4/6] cleanup constructors --- .../java/org/jlab/io/clara/Clas12Writer.java | 7 +- .../jlab/analysis/postprocess/Processor.java | 71 ++++++------------- 2 files changed, 25 insertions(+), 53 deletions(-) diff --git a/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java b/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java index 6a36e60488..a094b939cd 100644 --- a/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java +++ b/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java @@ -2,6 +2,7 @@ import java.io.File; import java.nio.file.Path; +import java.util.List; import java.util.TreeMap; import java.util.TreeSet; import org.jlab.analysis.postprocess.Processor; @@ -145,9 +146,9 @@ private Event getUnixEvent(Bank config) { */ private void postprocess() { int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0); - HelicitySequenceDelayed h = new HelicitySequenceDelayed(d); - h.addStream(helicities); - Processor p = new Processor(fullSchema, h, scalers); + HelicitySequenceDelayed helicity = new HelicitySequenceDelayed(d); + helicity.addStream(helicities); + Processor p = new Processor(List.of(filename), fullSchema, helicity, scalers); HipoReader r = new HipoReader(); r.open(filename); Event e = new Event(); diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java index a9ac84af8e..089a6ff312 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java @@ -1,10 +1,6 @@ package org.jlab.analysis.postprocess; import java.io.File; -import java.nio.file.FileSystems; -import java.nio.file.PathMatcher; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,55 +43,30 @@ public class Processor { private HelicitySequenceDelayed helicitySequence = null; private TreeMap eventUnix = null; - public Processor(File file, boolean restream, boolean rebuild) { - configure(Arrays.asList(file.getAbsolutePath()), restream, rebuild); - } - - public Processor(String dir, boolean restream, boolean rebuild) { - configure(findPreloadFiles(dir,DEF_PRELOAD_GLOB), restream, rebuild); - } - - public Processor(String dir, String glob, boolean restream, boolean rebuild) { - configure(findPreloadFiles(dir,glob), restream, rebuild); - } - public Processor(List files, boolean restream, boolean rebuild) { - configure(files, restream, rebuild); - } - - private void configure(List preloadFiles, boolean restream, boolean rebuild) { - if (!preloadFiles.isEmpty()) { - HipoReader r = new HipoReader(); - r.open(preloadFiles.get(0)); - schemaFactory = r.getSchemaFactory(); - r.close(); - runConfig = new Bank(schemaFactory.getSchema("RUN::config")); - recEvent = new Bank(schemaFactory.getSchema("REC::Event")); - conman = new ConstantsManager(); - conman.init(CCDB_TABLES); - helicitySequence = Util.getHelicity(preloadFiles, schemaFactory, restream, conman); - if (rebuild) chargeSequence = DaqScalersSequence.rebuildSequence(1, conman, preloadFiles); - else chargeSequence = DaqScalersSequence.readSequence(preloadFiles); - eventUnix = getEventUnixMap(schemaFactory, preloadFiles); - } + HipoReader r = new HipoReader(); + r.open(files.get(0)); + schemaFactory = r.getSchemaFactory(); + r.close(); + runConfig = new Bank(schemaFactory.getSchema("RUN::config")); + recEvent = new Bank(schemaFactory.getSchema("REC::Event")); + conman = new ConstantsManager(); + conman.init(CCDB_TABLES); + helicitySequence = Util.getHelicity(files, schemaFactory, restream, conman); + if (rebuild) chargeSequence = DaqScalersSequence.rebuildSequence(1, conman, files); + else chargeSequence = DaqScalersSequence.readSequence(files); + eventUnix = getEventUnixMap(schemaFactory, files); } - /** - * Get a list of files to preload, from one directory and a glob. - * @param dir - * @param glob - * @return list of preload files - */ - private static List findPreloadFiles(String dir, String glob) { - List ret = new ArrayList<>(); - if (dir != null) { - PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:"+dir+"/"+glob); - for (File f : (new File(dir)).listFiles()) { - if (matcher.matches(f.toPath())) - ret.add(f.getPath()); - } - } - return ret; + public Processor(List files, SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequence s) { + schemaFactory = schema; + helicitySequence = h; + chargeSequence = s; + runConfig = new Bank(schemaFactory.getSchema("RUN::config")); + recEvent = new Bank(schemaFactory.getSchema("REC::Event")); + conman = new ConstantsManager(); + conman.init(CCDB_TABLES); + eventUnix = getEventUnixMap(schemaFactory, files); } /** From 0fb7233880fdc26ecc58567f394694d49622b4c4 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 22 May 2026 18:35:02 -0400 Subject: [PATCH 5/6] process all events --- .../src/main/java/org/jlab/analysis/postprocess/Processor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java index 089a6ff312..112705d7c2 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java @@ -299,7 +299,6 @@ public static void main(String args[]) { for (String f : o.getInputList()) { HipoReader reader = new HipoReader(); - reader.setTags(0); reader.open(f); Event event = new Event(); while (reader.hasNext()) { From 28746d646b45a8b1aae71f68105618f066f8122a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 22 May 2026 19:08:22 -0400 Subject: [PATCH 6/6] cleanup/bugfix --- .../jlab/analysis/postprocess/Processor.java | 50 +++---------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java index 112705d7c2..7652b74d4c 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java @@ -1,9 +1,6 @@ package org.jlab.analysis.postprocess; -import java.io.File; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.TreeMap; import org.jlab.jnp.hipo4.data.Bank; @@ -31,9 +28,6 @@ public class Processor { public static final String CCDB_TABLES[] = {"/runcontrol/fcup","/runcontrol/slm", "/runcontrol/helicity","/daq/config/scalers/dsc1","/runcontrol/hwp"}; - public static final String DEF_PRELOAD_GLOB = "*.{hipo,h5}"; - - private final String outputPrefix = "tmp_"; private Bank runConfig = null; private Bank recEvent = null; @@ -76,20 +70,18 @@ public Processor(List files, SchemaFactory schema, HelicitySequenceDelay * @return map */ public static TreeMap getEventUnixMap(SchemaFactory schema, List files) { - Bank unix = new Bank(schema.getSchema("RUN::unix")); TreeMap m = new TreeMap<>(); Event e = new Event(); + Bank b = schema.getBank("RUN::unix");//new Bank(schema.getSchema("RUN::unix")); for (String f : files) { HipoReader r = new HipoReader(); r.setTags(1); r.open(f); while (r.hasNext()) { r.nextEvent(e); - e.read(unix); - int size = unix.getRows(); - for (int i=0; i0) { @@ -239,36 +231,6 @@ public void processEvent(Event event) { } } - /** - * Create rebuilt files from preload files. - * @param files - * @return map of rebuilt:preload files - */ - private Map rebuild(String dir, List files) { - File d = new File(dir); - if (!d.canWrite()) { - throw new RuntimeException("No write permissions on "+dir); - } - Map rebuiltFiles = new HashMap<>(); - for (String preloadFile : files) { - String rebuiltFile = dir+"/"+outputPrefix+preloadFile.replace(dir+"/",""); - Util.rebuildScalers(conman, preloadFile, rebuiltFile); - rebuiltFiles.put(rebuiltFile,preloadFile); - } - return rebuiltFiles; - } - - /** - * Replace files with new ones. - * @param files map of new:old filenames - */ - private static void replace(Map files) { - for (String rebuiltFile : files.keySet()) { - new File(files.get(rebuiltFile)).delete(); - new File(rebuiltFile).renameTo(new File(files.get(rebuiltFile))); - } - } - /** * The "postprocess" program. * @param args