Skip to content

Commit 35d7d33

Browse files
TejaX-AlaghariTejaX-Alaghari
authored
[SYCLomatic][Feature][PyTorch migration rule] Add python_build_script_migration_rule_pytorch.yaml rule file for Python script migration using PyTorch's XPU class instead of IPEX's XPU (#2564)
Aslo, Refactor --migrate-build-script and --migrate-build-script-only behavior for clarity. --migrate-build-script-only now migrates only CMake scripts with a warning. Added support for explicit combinations for build scripts migration: --migrate-build-script-only --migrate-build-script=<CMake|Python|CMake,Python> to migrate CMake, Python and both build script types respectively --migrate-build-script=<CMake|Python|CMake,Python> to migrate CMake, Python and both build script types including source files respectively.
1 parent 3a7fdda commit 35d7d33

42 files changed

Lines changed: 671 additions & 225 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎clang/include/clang/DPCT/DPCTOptions.inc‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -462,21 +462,27 @@ DPCT_ENUM_OPTION(
462462
llvm::cl::Optional)
463463

464464
DPCT_ENUM_OPTION(
465-
clang::dpct::opt, BuildScriptKind, BuildScript,
465+
clang::dpct::bits, BuildScriptKind, BuildScript,
466466
clang::dpct::DpctOptionClass::OC_Attribute,
467-
DPCT_OPTION_ACTIONS(clang::dpct::DpctActionKind::DAK_Migration,
467+
DPCT_OPTION_ACTIONS(clang::dpct::DpctActionKind::DAK_BuildScript,
468+
clang::dpct::DpctActionKind::DAK_Migration,
468469
clang::dpct::DpctActionKind::DAK_Analysis),
469470
"migrate-build-script",
470471
DPCT_OPTION_VALUES(
471-
DPCT_OPTION_ENUM_VALUE("CMake", int(BuildScriptKind::BS_Cmake),
472+
DPCT_OPTION_ENUM_VALUE("CMake", int(BuildScriptKind::BS_CMake),
472473
"Migrate the CMake file(s).", false),
473474
DPCT_OPTION_ENUM_VALUE("Python", int(BuildScriptKind::BS_Python),
474475
"Migrate the python build script file(s) of "
475476
"PyTorch based project.",
476477
false)),
477-
llvm::cl::desc("EXPERIMENTAL: Migrate build script(s).\n"),
478-
llvm::cl::init(BuildScriptKind::BS_None), llvm::cl::value_desc("value"),
479-
llvm::cl::Optional)
478+
llvm::cl::desc(
479+
"EXPERIMENTAL: A comma-separated list of build script type(s) to be "
480+
"migrated.\n"
481+
"By default, no build script is migrated.\n"
482+
"The values are:\n"),
483+
llvm::cl::CommaSeparated, llvm::cl::value_desc("value"),
484+
llvm::cl::cat(CtHelpCatAll), llvm::cl::cat(CtHelpCatAdvanced),
485+
llvm::cl::Optional, llvm::cl::ZeroOrMore)
480486

481487
DPCT_ENUM_OPTION(
482488
clang::dpct::opt, FormatRange, FormatRng,
@@ -856,7 +862,9 @@ DPCT_FLAG_OPTION(
856862
DPCT_OPTION_ACTIONS(clang::dpct::DpctActionKind::DAK_BuildScript),
857863
"migrate-build-script-only",
858864
llvm::cl::desc(
859-
"EXPERIMENTAL: Only migrate the build script(s). Default: off."))
865+
"EXPERIMENTAL: Only migrate the build script(s). Default: off."),
866+
llvm::cl::cat(CtHelpCatAll), llvm::cl::cat(CtHelpCatBasic),
867+
llvm::cl::Optional)
860868

861869
DPCT_OPTION(
862870
clang::dpct::list, std::string, ExcludePathList,

‎clang/lib/DPCT/AnalysisInfo.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2393,7 +2393,7 @@ std::string DpctGlobalInfo::SYCLHeaderExtension = std::string();
23932393
clang::tooling::UnifiedPath DpctGlobalInfo::CudaPath;
23942394
std::string DpctGlobalInfo::RuleFile = std::string();
23952395
UsmLevel DpctGlobalInfo::UsmLvl = UsmLevel::UL_None;
2396-
BuildScriptKind DpctGlobalInfo::BuildScriptVal = BuildScriptKind::BS_None;
2396+
unsigned DpctGlobalInfo::BuildScriptType = 0;
23972397
clang::CudaVersion DpctGlobalInfo::SDKVersion = clang::CudaVersion::UNKNOWN;
23982398
bool DpctGlobalInfo::NeedDpctDeviceExt = false;
23992399
bool DpctGlobalInfo::IsIncMigration = true;

‎clang/lib/DPCT/AnalysisInfo.h‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -832,8 +832,17 @@ class DpctGlobalInfo {
832832
static unsigned int getKCIndentWidth();
833833
static UsmLevel getUsmLevel() { return UsmLvl; }
834834
static void setUsmLevel(UsmLevel UL) { UsmLvl = UL; }
835-
static BuildScriptKind getBuildScript() { return BuildScriptVal; }
836-
static void setBuildScript(BuildScriptKind BSVal) { BuildScriptVal = BSVal; }
835+
static unsigned getBuildScript() { return BuildScriptType; }
836+
static void setBuildScript(unsigned BS_type) { BuildScriptType = BS_type; }
837+
template <BuildScriptKind BS_kind> static bool getMigrateBuildScriptType() {
838+
return BuildScriptType & (1 << static_cast<unsigned>(BS_kind));
839+
}
840+
static bool migrateCMakeScripts() {
841+
return getMigrateBuildScriptType<BuildScriptKind::BS_CMake>();
842+
}
843+
static bool migratePythonScripts() {
844+
return getMigrateBuildScriptType<BuildScriptKind::BS_Python>();
845+
}
837846
static clang::CudaVersion getSDKVersion() { return SDKVersion; }
838847
static void setSDKVersion(clang::CudaVersion V) { SDKVersion = V; }
839848
static bool isIncMigration() { return IsIncMigration; }
@@ -1570,7 +1579,7 @@ class DpctGlobalInfo {
15701579
static clang::tooling::UnifiedPath CudaPath;
15711580
static std::string RuleFile;
15721581
static UsmLevel UsmLvl;
1573-
static BuildScriptKind BuildScriptVal;
1582+
static unsigned BuildScriptType;
15741583
static clang::CudaVersion SDKVersion;
15751584
static bool NeedDpctDeviceExt;
15761585
static bool IsIncMigration;

‎clang/lib/DPCT/CommandOption/ValidateArguments.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ enum class OutputVerbosityLevel {
3838
OVL_Detailed,
3939
OVL_Diagnostics
4040
};
41-
enum class BuildScriptKind { BS_None, BS_Cmake, BS_Python };
41+
enum class BuildScriptKind { BS_CMake = 0, BS_Python };
4242
enum class DPCTFormatStyle { FS_LLVM, FS_Google, FS_Custom };
4343
enum class ReportFormatEnum { RFE_NotSetFormat, RFE_CSV, RFE_Formatted };
4444
enum class HelperFilesCustomizationLevel {

‎clang/lib/DPCT/DPCT.cpp‎

Lines changed: 78 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ const char *const CtHelpHint =
117117
const char *const CmakeScriptMigrationHelpHint =
118118
"Warning: CMake build script file like CMakeLists.txt is not found, so no CMake build script file will be migrated.";
119119

120-
const char *const SetupScriptMigrationHelpHint =
120+
const char *const PythonBuildScriptMigrationHelpHint =
121121
"Warning: No Python file is found, so no Python build script file will be migrated.";
122122

123123
const char *const BuildScriptMigrationHelpHint =
@@ -582,28 +582,43 @@ void checkIncMigrationOrExit() {
582582
}
583583
}
584584

585-
int migrateCmakeScript(const clang::tooling::UnifiedPath &InRoot,
586-
const clang::tooling::UnifiedPath &OutRoot) {
587-
if (!cmakeScriptNotFound()) {
588-
runWithCrashGuard(
589-
[&]() { doCmakeScriptMigration(InRoot, OutRoot); },
590-
"Error: dpct internal error. Migrating CMake scripts in \"" +
591-
InRootPath.getCanonicalPath().str() +
592-
"\" causing the error skipped. Migration continues.\n");
585+
int migrateBuildScripts(const clang::tooling::UnifiedPath &InRoot,
586+
const clang::tooling::UnifiedPath &OutRoot) {
587+
if (cmakeScriptNotFound() && pythonBuildScriptNotFound()) {
588+
std::cout << BuildScriptMigrationHelpHint << "\n";
589+
} else {
590+
if (DpctGlobalInfo::migrateCMakeScripts()) {
591+
if (!cmakeScriptNotFound()) {
592+
runWithCrashGuard(
593+
[&]() { doCmakeScriptMigration(InRoot, OutRoot); },
594+
"Error: dpct internal error. Migrating CMake scripts in \"" +
595+
InRootPath.getCanonicalPath().str() +
596+
"\" causing the error skipped. Migration continues.\n");
597+
} else {
598+
std::cout << CmakeScriptMigrationHelpHint << "\n";
599+
}
600+
}
601+
602+
if (DpctGlobalInfo::migratePythonScripts()) {
603+
if (pythonMigrationRulesRegistered() && !pythonBuildScriptNotFound()) {
604+
runWithCrashGuard(
605+
[&]() { doPythonBuildScriptMigration(InRoot, OutRoot); },
606+
"Error: dpct internal error. Migrating Python build scripts in \"" +
607+
InRoot.getCanonicalPath().str() +
608+
"\" causing the error skipped. Migration continues.\n");
609+
} else if (pythonBuildScriptNotFound()) {
610+
std::cout << PythonBuildScriptMigrationHelpHint << "\n";
611+
}
612+
}
593613
}
614+
594615
return MigrationSucceeded;
595616
}
596617

597-
int migratePythonScript(const clang::tooling::UnifiedPath &InRoot,
598-
const clang::tooling::UnifiedPath &OutRoot) {
599-
if (!pythonBuildScriptNotFound()) {
600-
runWithCrashGuard(
601-
[&]() { doPythonBuildScriptMigration(InRoot, OutRoot); },
602-
"Error: dpct internal error. Migrating Python build scripts in \"" +
603-
InRoot.getCanonicalPath().str() +
604-
"\" causing the error skipped. Migration continues.\n");
605-
}
606-
return MigrationSucceeded;
618+
void doBuildScriptMigration() {
619+
loadMainSrcFileInfo(OutRootPath);
620+
collectBuildScripts(InRootPath, OutRootPath);
621+
migrateBuildScripts(InRootPath, OutRootPath);
607622
}
608623

609624
// print APIMapping of Query
@@ -846,17 +861,21 @@ int runDPCT(int argc, const char **argv) {
846861
}
847862
}
848863

849-
if ((BuildScript == BuildScriptKind::BS_Cmake ||
850-
BuildScript == BuildScriptKind::BS_Python) &&
851-
!OptParser->getSourcePathList().empty()) {
852-
ShowStatus(MigrateBuildScriptIncorrectUse);
853-
dpctExit(MigrateBuildScriptIncorrectUse);
854-
}
855-
if ((BuildScript == BuildScriptKind::BS_Cmake ||
856-
BuildScript == BuildScriptKind::BS_Python) &&
857-
MigrateBuildScriptOnly) {
858-
ShowStatus(MigrateBuildScriptAndMigrateBuildScriptOnlyBothUse);
859-
dpctExit(MigrateBuildScriptAndMigrateBuildScriptOnlyBothUse);
864+
DpctGlobalInfo::setBuildScript(BuildScript.getBits());
865+
bool BuildScriptsSpecified = DpctGlobalInfo::migrateCMakeScripts() ||
866+
DpctGlobalInfo::migratePythonScripts();
867+
if (MigrateBuildScriptOnly) {
868+
if (!BuildScriptsSpecified) {
869+
llvm::errs() << getBuildScriptNotSpecifiedWarning();
870+
auto CMakeSelectionBits = 1 << (unsigned)BuildScriptKind::BS_CMake;
871+
DpctGlobalInfo::setBuildScript(CMakeSelectionBits);
872+
BuildScriptsSpecified = true;
873+
}
874+
} else {
875+
if (BuildScriptsSpecified && !OptParser->getSourcePathList().empty()) {
876+
ShowStatus(MigrateBuildScriptIncorrectUse);
877+
dpctExit(MigrateBuildScriptIncorrectUse);
878+
}
860879
}
861880

862881
int SDKIncPathRes = checkSDKPathOrIncludePath(CudaIncludePath);
@@ -1030,9 +1049,8 @@ int runDPCT(int argc, const char **argv) {
10301049
Tool.setPrintErrorMessage(false);
10311050
} else {
10321051
IsUsingDefaultOutRoot = OutRootPath.getPath().empty();
1033-
bool NeedCheckOutRootEmpty = !(BuildScript == BuildScriptKind::BS_Cmake ||
1034-
BuildScript == BuildScriptKind::BS_Python) &&
1035-
!MigrateBuildScriptOnly;
1052+
bool NeedCheckOutRootEmpty =
1053+
!(MigrateBuildScriptOnly || BuildScriptsSpecified);
10361054
if (!DpctGlobalInfo::isAnalysisModeEnabled() && IsUsingDefaultOutRoot &&
10371055
!getDefaultOutRoot(OutRootPath, NeedCheckOutRootEmpty) && !EnableCodePin) {
10381056
ShowStatus(MigrationErrorInvalidInRootOrOutRoot);
@@ -1133,7 +1151,6 @@ int runDPCT(int argc, const char **argv) {
11331151
DpctGlobalInfo::setKeepOriginCode(KeepOriginalCode);
11341152
DpctGlobalInfo::setSyclNamedLambda(SyclNamedLambda);
11351153
DpctGlobalInfo::setUsmLevel(USMLevel);
1136-
DpctGlobalInfo::setBuildScript(BuildScript);
11371154
// When enable codepin feature, the incremental migration will be disabled.
11381155
DpctGlobalInfo::setIsIncMigration(!NoIncrementalMigration && !EnableCodePin &&
11391156
!MigrateBuildScriptOnly);
@@ -1186,8 +1203,7 @@ int runDPCT(int argc, const char **argv) {
11861203
clang::dpct::initHeaderSpellings();
11871204

11881205
// load user defind rules in case.
1189-
if (MigrateBuildScriptOnly ||
1190-
DpctGlobalInfo::getBuildScript() == BuildScriptKind::BS_Cmake) {
1206+
if (DpctGlobalInfo::migrateCMakeScripts()) {
11911207
SmallString<128> FilePath1(DpctInstallPath.getCanonicalPath());
11921208
llvm::sys::path::append(FilePath1,
11931209
Twine("extensions/cmake_rules/"
@@ -1204,18 +1220,24 @@ int runDPCT(int argc, const char **argv) {
12041220
dpct::genCmakeHelperFunction(dpct::DpctGlobalInfo::getOutRoot());
12051221
}
12061222

1207-
if (MigrateBuildScriptOnly ||
1208-
DpctGlobalInfo::getBuildScript() == BuildScriptKind::BS_Python) {
1209-
SmallString<128> PythonRuleFilePath(DpctInstallPath.getCanonicalPath());
1210-
llvm::sys::path::append(
1211-
PythonRuleFilePath,
1212-
Twine("extensions/python_rules/"
1213-
"python_build_script_migration_rule_ipex.yaml"));
1214-
if (llvm::sys::fs::exists(PythonRuleFilePath)) {
1215-
std::vector<clang::tooling::UnifiedPath> PythonRuleFiles{
1216-
PythonRuleFilePath};
1217-
importRules(PythonRuleFiles);
1218-
// generage helper functions file in the outroot dir here
1223+
if (DpctGlobalInfo::migratePythonScripts()) {
1224+
// check if RuleFilePaths contains any user specified python migration rule
1225+
// file
1226+
bool pythonRuleFilePresent = std::any_of(
1227+
RuleFilePath.begin(), RuleFilePath.end(),
1228+
[](const clang::tooling::UnifiedPath &path) {
1229+
return path.getPath().contains("python_build_script_migration_rule");
1230+
});
1231+
1232+
if (!pythonRuleFilePresent) {
1233+
if (MigrateBuildScriptOnly) {
1234+
ShowStatus(
1235+
MigratePythonBuildScriptSpecifiedButPythonRuleFileNotSpecified);
1236+
dpctExit(
1237+
MigratePythonBuildScriptSpecifiedButPythonRuleFileNotSpecified);
1238+
}
1239+
1240+
llvm::errs() << getPythonRuleFileNotProvidedWarning();
12191241
}
12201242
}
12211243

@@ -1276,10 +1298,9 @@ int runDPCT(int argc, const char **argv) {
12761298
setValueToOptMap(clang::dpct::OPTION_UsmLevel,
12771299
static_cast<unsigned int>(DpctGlobalInfo::getUsmLevel()),
12781300
USMLevel.getNumOccurrences());
1279-
setValueToOptMap(
1280-
clang::dpct::OPTION_BuildScript,
1281-
static_cast<unsigned int>(DpctGlobalInfo::getBuildScript()),
1282-
BuildScript.getNumOccurrences());
1301+
setValueToOptMap(clang::dpct::OPTION_BuildScript,
1302+
DpctGlobalInfo::getBuildScript(),
1303+
BuildScript.getNumOccurrences());
12831304
setValueToOptMap(clang::dpct::OPTION_OptimizeMigration,
12841305
OptimizeMigration.getValue(),
12851306
OptimizeMigration.getNumOccurrences());
@@ -1301,14 +1322,7 @@ int runDPCT(int argc, const char **argv) {
13011322
}
13021323
// OC_Action: only migrate Build scripts.
13031324
if (MigrateBuildScriptOnly) {
1304-
loadMainSrcFileInfo(OutRootPath);
1305-
collectCmakeScriptsSpecified(OptParser, InRootPath, OutRootPath);
1306-
collectPythonBuildScriptsSpecified(OptParser, InRootPath, OutRootPath);
1307-
migrateCmakeScript(InRootPath, OutRootPath);
1308-
migratePythonScript(InRootPath, OutRootPath);
1309-
if (cmakeScriptNotFound() && pythonBuildScriptNotFound()) {
1310-
std::cout << BuildScriptMigrationHelpHint << "\n";
1311-
}
1325+
doBuildScriptMigration();
13121326
ShowStatus(MigrationBuildScriptCompleted);
13131327
dpctExit(MigrationSucceeded, false);
13141328
}
@@ -1439,23 +1453,10 @@ int runDPCT(int argc, const char **argv) {
14391453
OutRootPath.getCanonicalPath().str() +
14401454
"\" causing the error skipped. Migration continues.\n");
14411455

1442-
// OC_Action: Migrate CMake scripts after Code Migration
1443-
if (DpctGlobalInfo::getBuildScript() == BuildScriptKind::BS_Cmake) {
1444-
loadMainSrcFileInfo(OutRootPath);
1445-
collectCmakeScripts(InRootPath, OutRootPath);
1446-
migrateCmakeScript(InRootPath, OutRootPath);
1447-
if (cmakeScriptNotFound()) {
1448-
std::cout << CmakeScriptMigrationHelpHint << "\n";
1449-
}
1450-
}
1451-
1452-
if (DpctGlobalInfo::getBuildScript() == BuildScriptKind::BS_Python) {
1453-
loadMainSrcFileInfo(OutRootPath);
1454-
collectPythonBuildScripts(InRootPath, OutRootPath);
1455-
migratePythonScript(InRootPath, OutRootPath);
1456-
if (pythonBuildScriptNotFound()) {
1457-
std::cout << SetupScriptMigrationHelpHint << "\n";
1458-
}
1456+
// OC_Action: Migrate CMake/Python build scripts after Code Migration
1457+
if (DpctGlobalInfo::migrateCMakeScripts() ||
1458+
DpctGlobalInfo::migratePythonScripts()) {
1459+
doBuildScriptMigration();
14591460
}
14601461

14611462
ShowStatus(Status);

‎clang/lib/DPCT/ErrorHandle/Error.cpp‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -203,16 +203,20 @@ void ShowStatus(int Status, std::string Message) {
203203
StatusString = "Error: option '-migrate-build-script' is only used for "
204204
"whole project code migration.";
205205
break;
206-
case MigrateBuildScriptAndMigrateBuildScriptOnlyBothUse:
207-
StatusString = "Error: option '-migrate-build-script' and "
208-
"'-migrate-build-script-only' cannot be used together.";
209-
break;
210206
case MigrationErrorNoExplicitInRootAndBuildScript:
211207
StatusString =
212208
"Error: The option -migrate-build-script-only requires that either "
213209
"the option '--in-root' or the CMake or Python build file(s) be "
214210
"specified explicitly.";
215211
break;
212+
case MigratePythonBuildScriptSpecifiedButPythonRuleFileNotSpecified:
213+
StatusString =
214+
"Error: The option -migrate-build-script=Python requires python "
215+
"migration rules to be explicitly loaded with option --rule-file. For "
216+
"example, --rule-file={dpct_install_folder}/extensions/python_rules/"
217+
"python_build_script_migration_rule_[ipex|pytorch].yaml to load the "
218+
"predefined rules.";
219+
break;
216220
case MigrationErrorInvalidInstallPath:
217221
StatusString = "Error: " + Message + " not found.";
218222
break;
@@ -246,6 +250,14 @@ std::string getCheckVersionFailWarning() {
246250
"overview.html for more "
247251
"details.\n";
248252
}
253+
std::string getBuildScriptNotSpecifiedWarning() {
254+
return "Warning: Only CMake scripts will be migrated as no "
255+
"--migrate-build-script option is provided.\n";
256+
}
257+
std::string getPythonRuleFileNotProvidedWarning() {
258+
return "Warning: Rule file for python build script migration not found. "
259+
"Migration continues with python build script migration disabled.";
260+
}
249261

250262
bool IsUsingDefaultOutRoot = false;
251263

‎clang/lib/DPCT/ErrorHandle/Error.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,20 @@ enum ProcessStatus {
6363
MigrationErrorBuildScriptPathInvalid = -48,
6464
MigrateBuildScriptOnlyNotSpecifed = -49,
6565
MigrateBuildScriptIncorrectUse = -50,
66-
MigrateBuildScriptAndMigrateBuildScriptOnlyBothUse = -51,
67-
MigrationErrorNoExplicitInRootAndBuildScript = -52,
68-
MigrationErrorCannotWrite = -53,
69-
MigratePythonBuildScriptOnlyNotSpecifed = -54,
70-
};
66+
MigrationErrorNoExplicitInRootAndBuildScript = -51,
67+
MigrationErrorCannotWrite = -52,
68+
MigratePythonBuildScriptOnlyNotSpecifed = -53,
69+
MigratePythonBuildScriptSpecifiedButPythonRuleFileNotSpecified = -54,
70+
};
7171

7272
namespace clang {
7373
namespace dpct {
7474

7575
void ShowStatus(int Status, std::string Message = "");
7676
std::string getLoadYamlFailWarning(const clang::tooling::UnifiedPath& YamlPath);
7777
std::string getCheckVersionFailWarning();
78+
std::string getBuildScriptNotSpecifiedWarning();
79+
std::string getPythonRuleFileNotProvidedWarning();
7880

7981
extern bool IsUsingDefaultOutRoot;
8082
void removeDefaultOutRootFolder(const clang::tooling::UnifiedPath &DefaultOutRoot);

0 commit comments

Comments
 (0)