Skip to content

Commit a6c68c0

Browse files
t-bt-b
committed
Speedup parsing using regexps
Since the initial introduction of extendend parsing in 12e38b9 (regex parsing, 2015-12-30) the speed of parsing the procedure files is not good. A quick check with FunctionProfiling reveals Total time: 25.3811, Time in Function code: 18.0965 (71.3%) Top function percentages: Function CodeBrowser.ipf [CodeBrowserModule] addDecoratedConstants: 23% Function CodeBrowser.ipf [CodeBrowserModule] addDecoratedMenu: 23% Function CodeBrowser.ipf [CodeBrowserModule] addDecoratedStructure: 23% Function CodeBrowser.ipf [CodeBrowserModule] addDecoratedMacros: 22% Annotated Top Functions: ******************************************************************************************* Function: CodeBrowser.ipf [CodeBrowserModule] addDecoratedConstants; Percent total 23% ******************************************************************************************* [00] |Function addDecoratedConstants(module, procedureWithoutModule, declWave, lineWave) [00] | String module, procedureWithoutModule [00] | WAVE/T declWave [00] | WAVE/D lineWave [00] | [00] | Variable numLines, i, idx, numEntries, numMatches [00] | String procText, re, def, name [00] | [00] | Wave/T helpWave = getHelpWave() [00] | [00] | // get procedure code [00] | procText = getProcedureText("", 0, module, procedureWithoutModule) [00]* | numLines = ItemsInList(procText, "\r") [00] | [00] | // search code and return wavLineNumber [22]** | Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r") [00] | // help for regex on https://regex101.com/ [00] | re = "^(?i)[[:space:]]*((?:override\s+)?(?:static)?[[:space:]]*(?:Str)?Constant)[[:space:]]+([^=\s]*)\s*=\s*(?:\"(?:[^\"\\\\]|\\\\.)+\"|0[xX][0-9a-fA-F]+|[0-9]+)\s*(?:[\/]{2}.*)?" [00]* | Grep/Q/INDX/E=re text [00] | Wave W_Index [00]* | Duplicate/FREE W_Index wavLineNumber [00]* | KillWaves/Z W_Index [00] | KillStrings/Z S_fileName [00] | WaveClear W_Index [00] | if(!V_Value) // no matches [00] | return 0 [00] | endif [00] | [00]* | numMatches = DimSize(wavLineNumber, 0) [00] | numEntries = DimSize(declWave, 0) [00]* | Redimension/N=(numEntries + numMatches, -1) declWave, lineWave, helpWave [00] | [00] | idx = numEntries [00]* | for(i = 0; i < numMatches; i += 1) [00]* | SplitString/E=re text[wavLineNumber[i]], def, name [00] | [00]* | declWave[idx][0] = createMarkerForType(LowerStr(def)) [00] | declWave[idx][1] = name [00] | lineWave[idx] = wavLineNumber[i] [00] | idx += 1 [00] | endfor [00] | [00] | KillWaves/Z W_Index [00] |End ******************************************************************************************* Function: CodeBrowser.ipf [CodeBrowserModule] addDecoratedMenu; Percent total 23% ******************************************************************************************* [00] |Function addDecoratedMenu(module, procedureWithoutModule, declWave, lineWave) [00] | String module, procedureWithoutModule [00] | WAVE/T declWave [00] | WAVE/D lineWave [00] | [00] | Variable numLines, idx, numEntries, numMatches [00] | String procText, re, def, name, type [00] | String currentMenu = "" [00] | [00] | Wave/T helpWave = getHelpWave() [00] | [00] | // get procedure code [00] | procText = getProcedureText("", 0, module, procedureWithoutModule) [00]* | numLines = ItemsInList(procText, "\r") [00] | [00] | // search code and return wavLineNumber [23]** | Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r") [00] | // regexp: match case insensitive (?i) spaces don't matter. search for menu or submenu with a name in double quotes. [00] | // help for regex on https://regex101.com/ [00] | re = "^(?i)[[:space:]]*(menu|submenu)[[:space:]]+\"((?:[^\"\\\\]|\\\\.)+)\"(?:[[:space:]]*[,][[:space:]]*(hideable|dynamic|contextualmenu))?" [00]* | Grep/Q/INDX/E=re text [00] | numMatches = !!V_Value [00] | Wave W_Index [00]* | Duplicate/FREE W_Index wavLineNumber [00]* | KillWaves/Z W_Index [00]* | KillStrings/Z S_filename [00] | if(!numMatches) [00] | return 0 [00] | endif [00] | [00] | numMatches = DimSize(wavLineNumber, 0) [00] | numEntries = DimSize(declWave, 0) [00] | Redimension/N=(numEntries + numMatches, -1) declWave, lineWave, helpWave [00] | [00] | for(idx = numEntries; idx < (numEntries + numMatches); idx += 1) [00]* | SplitString/E=re text[wavLineNumber[(idx - numEntries)]], def, name, type [00] | def = LowerStr(def) [00] | if(!cmpstr(def, "menu")) [00] | currentMenu = name [00] | endif [00] | declWave[idx][0] = createMarkerForType(def) [00] | declWave[idx][1] = "Menu " + currentMenu [00] | if(!cmpstr(def, "submenu")) [00] | declWave[idx][1] += ":" + name [00] | endif [00] | if(cmpstr(type, "")) [00] | declWave[idx][1] += "(" + type + ")" [00] | endif [00] | [00] | lineWave[idx] = wavLineNumber[(idx - numEntries)] [00] | endfor [00] |End ******************************************************************************************* Function: CodeBrowser.ipf [CodeBrowserModule] addDecoratedStructure; Percent total 23% ******************************************************************************************* [00] |Function addDecoratedStructure(module, procedureWithoutModule, declWave, lineWave, [parseVariables]) [00] | String module, procedureWithoutModule [00] | WAVE/T declWave [00] | WAVE/D lineWave [00] | Variable parseVariables [00]* | if(paramIsDefault(parseVariables) | parseVariables != 1) [00] | parseVariables = 1 // added for debugging [00] | endif [00] | [00] | variable numLines, idx, numEntries, numMatches [00] | string procText, reStart, reEnd, name, StaticKeyword [00] | [00] | Wave/T helpWave = getHelpWave() [00] | [00] | // get procedure code [00] | procText = getProcedureText("", 0, module, procedureWithoutModule) [00]* | numLines = ItemsInList(procText, "\r") [00] | if(numLines == 0) [00] | debugPrint("no Content in Procedure " + procedureWithoutModule) [00] | endif [00] | [00] | // search code and return wavLineNumber [22]** | Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r") [00] | // regexp: match case insensitive (?i) leading spaces don't matter. optional static statement. search for structure name which contains no spaces. followed by an optional space and nearly anything like inline comments [00] | // help for regex on https://regex101.com/ [00] | reStart = "^(?i)[[:space:]]*((?:static[[:space:]])?)[[:space:]]*structure[[:space:]]+([^[:space:]\/]+)[[:space:]\/]?.*" [00]* | Grep/Q/INDX/E=reStart text [00] | Wave W_Index [00]* | Duplicate/FREE W_Index wavStructureStart [00]* | KillWaves/Z W_Index [00]* | KillStrings/Z S_fileName [00] | WaveClear W_Index [00] | if(!V_Value) // no matches [00] | return 0 [00] | endif [00] | numMatches = DimSize(wavStructureStart, 0) [00] | [00] | // optionally analyze structure elements [00] | if(parseVariables) [00] | // regexp: match case insensitive endstructure followed by (space or /) and anything else or just a lineend [00] | // does not match endstructure23 but endstructure// [00] | reEnd = "^(?i)[[:space:]]*(?:endstructure(?:[[:space:]]|\/).*)|endstructure$" [00]* | Grep/Q/INDX/E=reEnd text [00] | Wave W_Index [00]* | Duplicate/FREE W_Index wavStructureEnd [00]* | KillWaves/Z W_Index [00] | KillStrings/Z S_fileName [00] | WaveClear W_Index [00] | if(numMatches != DimSize(wavStructureEnd, 0)) [00] | numMatches = 0 [00] | return 0 [00] | endif [00] | endif [00] | [00] | numEntries = DimSize(declWave, 0) [00] | Redimension/N=(numEntries + numMatches, -1) declWave, lineWave, helpWave [00] | [00] | for(idx = numEntries; idx < (numEntries + numMatches); idx +=1) [00]* | SplitString/E=reStart text[wavStructureStart[(idx - numEntries)]], StaticKeyword, name [00]* | declWave[idx][0] = createMarkerForType(LowerStr(StaticKeyword) + "structure") // no " " between static and structure needed [00] | declWave[idx][1] = name [00] | [00] | // optionally parse structure elements [00] | if(parseVariables) [00] | Duplicate/FREE/R=[(wavStructureStart[(idx - numEntries)]),(wavStructureEnd[(idx - numEntries)])] text, temp [00] | declWave[idx][1] += getStructureElements(temp) [00] | WaveClear temp [00] | endif [00] | [00]* | lineWave[idx] = wavStructureStart[(idx - numEntries)] [00] | endfor [00] | [00] | WaveClear wavStructureStart, wavStructureEnd [00] |End ******************************************************************************************* Function: CodeBrowser.ipf [CodeBrowserModule] addDecoratedMacros; Percent total 22% ******************************************************************************************* [00] |Function addDecoratedMacros(module, procedureWithoutModule, declWave, lineWave) [00] | String module, procedureWithoutModule [00] | WAVE/T declWave [00] | WAVE/D lineWave [00] | [00] | Variable numLines, idx, numEntries, numMatches [00] | String procText, re, def, name, arguments, type [00] | [00] | Wave/T helpWave = getHelpWave() [00] | [00] | // get procedure code [00]* | procText = getProcedureText("", 0, module, procedureWithoutModule) [00]* | numLines = ItemsInList(procText, "\r") [00] | [00] | // search code and return wavLineNumber [22]** | Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r") [00] | // regexp: match case insensitive (?i) spaces don't matter. search for window or macro or proc. Macro Name is the the next non-space character followed by brackets () where the arguments are. At the end there might be a colon, specifying the type of macro and a comment beginning with / [00] | // macro should have no arguments. Handled for backwards compatibility. [00] | // help for regex on https://regex101.com/ [00] | re = "^(?i)[[:space:]]*(window|macro|proc)[[:space:]]+([^[:space:]]+)[[:space:]]*\((.*)\)[[:space:]]*[:]?[[:space:]]*([^[:space:]\/]*).*" [00]* | Grep/Q/INDX/E=re text [00] | Wave W_Index [00]* | Duplicate/FREE W_Index wavLineNumber [00]* | KillWaves/Z W_Index [00]* | KillStrings/Z S_fileName [00] | WaveClear W_Index [00] | if(!V_Value) // no matches [00] | return 0 [00] | endif [00] | [00] | numMatches = DimSize(wavLineNumber, 0) [00] | numEntries = DimSize(declWave, 0) [00] | Redimension/N=(numEntries + numMatches, -1) declWave, lineWave, helpWave [00] | [00] | for(idx = numEntries; idx < (numEntries + numMatches); idx += 1) [00]* | SplitString/E=re text[wavLineNumber[(idx - numEntries)]], def, name, arguments, type [00] | // def containts window/macro/proc [00] | // type contains Panel/Layout for subclasses of window macros [00] | declWave[idx][0] = createMarkerForType(LowerStr(def)) [00] | declWave[idx][1] = name + "(" + trimArgument(arguments, ",", strListSepStringOutput = ", ") + ")" + " : " + type [00] | lineWave[idx] = wavLineNumber[(idx - numEntries)] [00] | endfor [00] |End So pulling out the procedure text conversion to list and do it only once is a first obvious optimisation. For IP7 and later the new function ListToTextWave is also used. The parsing time for MIES_DAEphys.ipf dropped from 1.5s to 56ms.
1 parent 3c1f33f commit a6c68c0

File tree

1 file changed

+39
-39
lines changed

1 file changed

+39
-39
lines changed

‎procedures/CodeBrowser.ipf‎

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -373,8 +373,8 @@ Function addDecoratedFunctions(module, procedure, declWave, lineWave)
373373
End
374374

375375
// Adds Constants/StrConstants by searching for them in the Procedure with a Regular Expression
376-
Function addDecoratedConstants(module, procedureWithoutModule, declWave, lineWave)
377-
String module, procedureWithoutModule
376+
Function addDecoratedConstants(text, declWave, lineWave)
377+
WAVE/T text
378378
WAVE/T declWave
379379
WAVE/D lineWave
380380

@@ -383,12 +383,6 @@ Function addDecoratedConstants(module, procedureWithoutModule, declWave, lineWav
383383

384384
Wave/T helpWave = getHelpWave()
385385

386-
// get procedure code
387-
procText = getProcedureText("", 0, module, procedureWithoutModule)
388-
numLines = ItemsInList(procText, "\r")
389-
390-
// search code and return wavLineNumber
391-
Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r")
392386
// help for regex on https://regex101.com/
393387
re = "^(?i)[[:space:]]*((?:override\s+)?(?:static)?[[:space:]]*(?:Str)?Constant)[[:space:]]+([^=\s]*)\s*=\s*(?:\"(?:[^\"\\\\]|\\\\.)+\"|0[xX][0-9a-fA-F]+|[0-9]+)\s*(?:[\/]{2}.*)?"
394388
Grep/Q/INDX/E=re text
@@ -418,8 +412,8 @@ Function addDecoratedConstants(module, procedureWithoutModule, declWave, lineWav
418412
KillWaves/Z W_Index
419413
End
420414

421-
Function addDecoratedMacros(module, procedureWithoutModule, declWave, lineWave)
422-
String module, procedureWithoutModule
415+
Function addDecoratedMacros(text, declWave, lineWave)
416+
WAVE/T text
423417
WAVE/T declWave
424418
WAVE/D lineWave
425419

@@ -428,12 +422,6 @@ Function addDecoratedMacros(module, procedureWithoutModule, declWave, lineWave)
428422

429423
Wave/T helpWave = getHelpWave()
430424

431-
// get procedure code
432-
procText = getProcedureText("", 0, module, procedureWithoutModule)
433-
numLines = ItemsInList(procText, "\r")
434-
435-
// search code and return wavLineNumber
436-
Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r")
437425
// regexp: match case insensitive (?i) spaces don't matter. search for window or macro or proc. Macro Name is the the next non-space character followed by brackets () where the arguments are. At the end there might be a colon, specifying the type of macro and a comment beginning with /
438426
// macro should have no arguments. Handled for backwards compatibility.
439427
// help for regex on https://regex101.com/
@@ -462,8 +450,8 @@ Function addDecoratedMacros(module, procedureWithoutModule, declWave, lineWave)
462450
endfor
463451
End
464452

465-
Function addDecoratedStructure(module, procedureWithoutModule, declWave, lineWave, [parseVariables])
466-
String module, procedureWithoutModule
453+
Function addDecoratedStructure(text, declWave, lineWave, [parseVariables])
454+
WAVE/T text
467455
WAVE/T declWave
468456
WAVE/D lineWave
469457
Variable parseVariables
@@ -476,15 +464,6 @@ Function addDecoratedStructure(module, procedureWithoutModule, declWave, lineWav
476464

477465
Wave/T helpWave = getHelpWave()
478466

479-
// get procedure code
480-
procText = getProcedureText("", 0, module, procedureWithoutModule)
481-
numLines = ItemsInList(procText, "\r")
482-
if(numLines == 0)
483-
debugPrint("no Content in Procedure " + procedureWithoutModule)
484-
endif
485-
486-
// search code and return wavLineNumber
487-
Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r")
488467
// regexp: match case insensitive (?i) leading spaces don't matter. optional static statement. search for structure name which contains no spaces. followed by an optional space and nearly anything like inline comments
489468
// help for regex on https://regex101.com/
490469
reStart = "^(?i)[[:space:]]*((?:static[[:space:]])?)[[:space:]]*structure[[:space:]]+([^[:space:]\/]+)[[:space:]\/]?.*"
@@ -537,8 +516,8 @@ Function addDecoratedStructure(module, procedureWithoutModule, declWave, lineWav
537516
WaveClear wavStructureStart, wavStructureEnd
538517
End
539518

540-
Function addDecoratedMenu(module, procedureWithoutModule, declWave, lineWave)
541-
String module, procedureWithoutModule
519+
Function addDecoratedMenu(text, declWave, lineWave)
520+
WAVE/T text
542521
WAVE/T declWave
543522
WAVE/D lineWave
544523

@@ -548,12 +527,6 @@ Function addDecoratedMenu(module, procedureWithoutModule, declWave, lineWave)
548527

549528
Wave/T helpWave = getHelpWave()
550529

551-
// get procedure code
552-
procText = getProcedureText("", 0, module, procedureWithoutModule)
553-
numLines = ItemsInList(procText, "\r")
554-
555-
// search code and return wavLineNumber
556-
Make/FREE/N=(numLines)/T text = StringFromList(p, procText, "\r")
557530
// regexp: match case insensitive (?i) spaces don't matter. search for menu or submenu with a name in double quotes.
558531
// help for regex on https://regex101.com/
559532
re = "^(?i)[[:space:]]*(menu|submenu)[[:space:]]+\"((?:[^\"\\\\]|\\\\.)+)\"(?:[[:space:]]*[,][[:space:]]*(hideable|dynamic|contextualmenu))?"
@@ -738,15 +711,42 @@ Function/S parseProcedure(procedure, [checksumIsCalculated])
738711
// scan and add elements to lists
739712
resetLists(decls, lines, helps)
740713
addDecoratedFunctions(procedure.module, procedure.fullName, decls, lines)
741-
addDecoratedConstants(procedure.module, procedure.name, decls, lines)
742-
addDecoratedMacros(procedure.module, procedure.name, decls, lines)
743-
addDecoratedStructure(procedure.module, procedure.name, decls, lines)
744-
addDecoratedMenu(procedure.module, procedure.name, decls, lines)
714+
715+
WAVE/T procContent = getProcedureTextAsWave(procedure.module, procedure.fullName)
716+
addDecoratedConstants(procContent, decls, lines)
717+
addDecoratedMacros(procContent, decls, lines)
718+
addDecoratedStructure(procContent, decls, lines)
719+
addDecoratedMenu(procContent, decls, lines)
745720

746721
// stop timer
747722
setParsingTime(timerStop(timer))
748723
End
749724

725+
/// @brief Return the text of the given procedure as free wave splitted at the EOL
726+
static Function/WAVE getProcedureTextAsWave(module, procedureWithoutModule)
727+
string module, procedureWithoutModule
728+
729+
string procText
730+
variable numLines
731+
732+
// get procedure code
733+
procText = getProcedureText("", 0, module, procedureWithoutModule)
734+
735+
#if (IgorVersion() >= 7.0)
736+
return ListToTextWave(procText, "\r")
737+
#else
738+
numLines = ItemsInList(procText, "\r")
739+
740+
if(numLines == 0)
741+
Make/FREE/N=(numLines)/T wv
742+
return wv
743+
endif
744+
745+
Make/FREE/N=(numLines)/T wv = StringFromList(p, procText, "\r")
746+
return wv
747+
#endif
748+
End
749+
750750
// Identifier = module#procedure
751751
static Function saveResults(procedure)
752752
STRUCT procedure &procedure

0 commit comments

Comments
 (0)