// Lot MAPDL1 (MAPDL1-SPEC-03, P1). Voir LPImportLegacyStreetMapCommandlet.h. #include "LPImportLegacyStreetMapCommandlet.h" #include "AssetRegistry/AssetRegistryModule.h" #include "HAL/FileManager.h" #include "Misc/FileHelper.h" #include "Misc/PackageName.h" #include "Misc/Paths.h" #include "StreetMap.h" #include "UObject/EnumProperty.h" #include "UObject/Package.h" #include "UObject/SavePackage.h" #include "UObject/UnrealType.h" DEFINE_LOG_CATEGORY_STATIC(LogLPImportLegacyStreetMap, Log, All); namespace LPLegacyStreetMap { /** Lecteur petit-boutiste borne : toute lecture hors du tampon pose bError et rend une valeur nulle. */ struct FReader { const TArray& Data; int64 Pos = 0; bool bError = false; FReader(const TArray& InData, int64 InPos) : Data(InData), Pos(InPos) {} bool Has(int64 Count) { if (bError || Count < 0 || Pos < 0 || Pos + Count > Data.Num()) { bError = true; return false; } return true; } template T Read() { T Value{}; if (Has(sizeof(T))) { FMemory::Memcpy(&Value, Data.GetData() + Pos, sizeof(T)); Pos += sizeof(T); } return Value; } void Skip(int64 Count) { if (Has(Count)) { Pos += Count; } } /** FString UE4 : longueur positive = ANSI, negative = UTF-16, terminateur nul compris. */ FString Str() { const int32 Len = Read(); FString Out; if (Len > 0 && Has(Len)) { for (int32 Index = 0; Index < Len - 1; ++Index) { Out.AppendChar(TCHAR(Data[Pos + Index])); } Pos += Len; } else if (Len < 0 && Has(int64(-Len) * 2)) { for (int32 Index = 0; Index < -Len - 1; ++Index) { Out.AppendChar(TCHAR(Data[Pos + 2 * Index] | (Data[Pos + 2 * Index + 1] << 8))); } Pos += int64(-Len) * 2; } return Out; } }; /** Etiquette de propriete UE4 (FPropertyTag, format 4.19-4.27). */ struct FTag { FString Name; FString Type; int32 Size = 0; int32 ArrayIndex = 0; FString StructName; FString InnerType; FString EnumName; uint8 BoolVal = 0; }; /** Etat de la lecture : table de noms et constats du rapport. */ struct FContext { TArray Names; TMap Unknown; TMap Mismatch; FString Error; FString Name(FReader& R) const { const int32 Index = R.Read(); const int32 Number = R.Read(); if (!Names.IsValidIndex(Index)) { R.bError = true; return FString(); } return Number == 0 ? Names[Index] : FString::Printf(TEXT("%s_%d"), *Names[Index], Number - 1); } bool Fail(FReader& R, const FString& Message) { if (Error.IsEmpty()) { Error = FString::Printf(TEXT("%s (octet %lld)"), *Message, R.Pos); } R.bError = true; return false; } }; /** Lit une etiquette ; faux pour "None" (fin de liste) ou sur erreur. */ bool ReadTag(FReader& R, FContext& Ctx, FTag& Out) { Out = FTag(); Out.Name = Ctx.Name(R); if (R.bError || Out.Name == TEXT("None")) { return false; } Out.Type = Ctx.Name(R); Out.Size = R.Read(); Out.ArrayIndex = R.Read(); if (Out.Type == TEXT("StructProperty")) { Out.StructName = Ctx.Name(R); R.Skip(16); } else if (Out.Type == TEXT("BoolProperty")) { Out.BoolVal = R.Read(); } else if (Out.Type == TEXT("ByteProperty") || Out.Type == TEXT("EnumProperty")) { Out.EnumName = Ctx.Name(R); } else if (Out.Type == TEXT("ArrayProperty") || Out.Type == TEXT("SetProperty")) { Out.InnerType = Ctx.Name(R); } else if (Out.Type == TEXT("MapProperty")) { Out.InnerType = Ctx.Name(R); Ctx.Name(R); } if (R.Read() != 0) { R.Skip(16); } return !R.bError && Out.Size >= 0; } bool ReadProperties(FReader& R, int64 Limit, const UStruct* Struct, void* Container, FContext& Ctx, const FString& Where); /** Valeur d'enumeration (nom court ou qualifie) -> valeur numerique. */ int64 EnumValue(const UEnum* Enum, const FString& Value) { if (!Enum) { return INDEX_NONE; } int64 Result = Enum->GetValueByNameString(Value); if (Result == INDEX_NONE && !Value.Contains(TEXT("::"))) { Result = Enum->GetValueByNameString(Enum->GetName() + TEXT("::") + Value); } return Result; } /** * Lit une valeur de type Type et l'ecrit dans Prop (ValuePtr) si les types sont compatibles ; Prop nul ou incompatible : * la valeur est consommee sans etre ecrite. Faux sur erreur de format. */ bool ReadValue(FReader& R, int64 Limit, const FTag& Tag, FProperty* Prop, void* ValuePtr, FContext& Ctx, const FString& Where) { auto Mismatch = [&]() { if (Prop) { Ctx.Mismatch.FindOrAdd(Where + TEXT(".") + Tag.Name + TEXT(" (") + Tag.Type + TEXT(")"))++; } }; const FString& Type = Tag.Type; if (Type == TEXT("BoolProperty")) { if (FBoolProperty* P = CastField(Prop)) { P->SetPropertyValue(ValuePtr, Tag.BoolVal != 0); } else { Mismatch(); } return true; } if (Type == TEXT("IntProperty") || Type == TEXT("FloatProperty") || Type == TEXT("DoubleProperty")) { const bool bInt = Type == TEXT("IntProperty"); const double AsDouble = bInt ? double(R.Read()) : (Type == TEXT("FloatProperty") ? double(R.Read()) : R.Read()); FNumericProperty* P = CastField(Prop); if (P && P->IsFloatingPoint()) { P->SetFloatingPointPropertyValue(ValuePtr, AsDouble); } else if (P && P->IsInteger() && bInt && !P->IsEnum()) { P->SetIntPropertyValue(ValuePtr, int64(AsDouble)); } else { Mismatch(); } return !R.bError; } if (Type == TEXT("StrProperty")) { const FString Value = R.Str(); if (FStrProperty* P = CastField(Prop)) { *static_cast(ValuePtr) = Value; } else { Mismatch(); } return !R.bError; } if (Type == TEXT("NameProperty")) { const FString Value = Ctx.Name(R); if (FNameProperty* P = CastField(Prop)) { *static_cast(ValuePtr) = FName(*Value); } else { Mismatch(); } return !R.bError; } if (Type == TEXT("ByteProperty") || Type == TEXT("EnumProperty")) { const bool bNamed = Type == TEXT("EnumProperty") || (!Tag.EnumName.IsEmpty() && Tag.EnumName != TEXT("None")); const FString Named = bNamed ? Ctx.Name(R) : FString(); const int64 Raw = bNamed ? INDEX_NONE : int64(R.Read()); if (R.bError) { return false; } if (FByteProperty* ByteProp = CastField(Prop)) { const int64 Value = bNamed ? EnumValue(ByteProp->Enum, Named) : Raw; if (Value == INDEX_NONE) { Ctx.Mismatch.FindOrAdd(Where + TEXT(".") + Tag.Name + TEXT(" = ") + Named)++; } else { ByteProp->SetPropertyValue(ValuePtr, uint8(Value)); } } else if (FEnumProperty* EnumProp = CastField(Prop)) { const int64 Value = bNamed ? EnumValue(EnumProp->GetEnum(), Named) : Raw; if (Value == INDEX_NONE) { Ctx.Mismatch.FindOrAdd(Where + TEXT(".") + Tag.Name + TEXT(" = ") + Named)++; } else { EnumProp->GetUnderlyingProperty()->SetIntPropertyValue(ValuePtr, Value); } } else { Mismatch(); } return true; } if (Type == TEXT("StructProperty")) { FStructProperty* P = CastField(Prop); if (Tag.StructName == TEXT("Vector2D") || Tag.StructName == TEXT("Vector")) { // Structures natives UE4 : flottants simples, sans etiquettes. const bool b2D = Tag.StructName == TEXT("Vector2D"); const float X = R.Read(), Y = R.Read(), Z = b2D ? 0.f : R.Read(); if (P && b2D && P->Struct == TBaseStructure::Get()) { *static_cast(ValuePtr) = FVector2D(X, Y); } else if (P && !b2D && P->Struct == TBaseStructure::Get()) { *static_cast(ValuePtr) = FVector(X, Y, Z); } else { Mismatch(); } return !R.bError; } if (P && P->Struct->GetName() != Tag.StructName) { Mismatch(); P = nullptr; } return ReadProperties(R, Limit, P ? P->Struct : nullptr, P ? ValuePtr : nullptr, Ctx, Where + TEXT(".") + Tag.StructName); } if (Type == TEXT("ArrayProperty")) { const int32 Count = R.Read(); if (R.bError || Count < 0 || Count > Limit - R.Pos) { return Ctx.Fail(R, FString::Printf(TEXT("tableau %s.%s : nombre %d invalide"), *Where, *Tag.Name, Count)); } FArrayProperty* P = CastField(Prop); if (Prop && !P) { Mismatch(); } FTag Inner; Inner.Name = Tag.Name; Inner.Type = Tag.InnerType; if (Tag.InnerType == TEXT("StructProperty")) { // Tableau de structures : une etiquette interieure, puis les elements. FTag ElementTag; if (!ReadTag(R, Ctx, ElementTag)) { return Ctx.Fail(R, FString::Printf(TEXT("tableau %s.%s : etiquette interieure illisible"), *Where, *Tag.Name)); } Inner.StructName = ElementTag.StructName; } else if (Tag.InnerType == TEXT("BoolProperty")) { // Booleens d'un tableau : un octet par element. Inner.Type = TEXT("ByteProperty"); } if (P) { FScriptArrayHelper Helper(P, ValuePtr); Helper.EmptyValues(); Helper.AddValues(Count); for (int32 Index = 0; Index < Count; ++Index) { if (!ReadValue(R, Limit, Inner, P->Inner, Helper.GetRawPtr(Index), Ctx, Where + TEXT(".") + Tag.Name)) { return false; } } } else { for (int32 Index = 0; Index < Count; ++Index) { if (!ReadValue(R, Limit, Inner, nullptr, nullptr, Ctx, Where + TEXT(".") + Tag.Name)) { return false; } } } return !R.bError; } return Ctx.Fail(R, FString::Printf(TEXT("type %s non pris en charge (%s.%s) dans un tableau"), *Type, *Where, *Tag.Name)); } /** Liste de proprietes etiquetees jusqu'a "None". Struct nul : tout est saute par la taille de l'etiquette. */ bool ReadProperties(FReader& R, int64 Limit, const UStruct* Struct, void* Container, FContext& Ctx, const FString& Where) { FTag Tag; while (ReadTag(R, Ctx, Tag)) { const int64 ValueStart = R.Pos; const int64 ValueEnd = ValueStart + Tag.Size; if (ValueEnd > Limit) { return Ctx.Fail(R, FString::Printf(TEXT("%s.%s : taille %d hors de l'export"), *Where, *Tag.Name, Tag.Size)); } FProperty* Prop = Struct ? FindFProperty(Struct, FName(*Tag.Name)) : nullptr; if (Struct && !Prop) { Ctx.Unknown.FindOrAdd(Where + TEXT(".") + Tag.Name)++; } if (Prop && Tag.ArrayIndex >= Prop->ArrayDim) { Ctx.Mismatch.FindOrAdd(Where + TEXT(".") + Tag.Name + TEXT(" (indice)"))++; Prop = nullptr; } if (Prop && Container) { if (!ReadValue(R, ValueEnd, Tag, Prop, Prop->ContainerPtrToValuePtr(Container, Tag.ArrayIndex), Ctx, Where)) { return false; } if (Tag.Type != TEXT("BoolProperty") && R.Pos != ValueEnd) { return Ctx.Fail(R, FString::Printf(TEXT("%s.%s : %lld octets lus pour %d annonces"), *Where, *Tag.Name, R.Pos - ValueStart, Tag.Size)); } } R.Pos = ValueEnd; } return !R.bError; } /** Entree de la table d'imports. */ struct FImport { FString ClassName; FString ObjectName; }; } ULPImportLegacyStreetMapCommandlet::ULPImportLegacyStreetMapCommandlet() { IsClient = false; IsServer = false; IsEditor = true; LogToConsole = true; } int32 ULPImportLegacyStreetMapCommandlet::Main(const FString& Params) { using namespace LPLegacyStreetMap; TArray Tokens; TArray Switches; TMap ParamsMap; ParseCommandLine(*Params, Tokens, Switches, ParamsMap); const FString UassetPath = ParamsMap.FindRef(TEXT("Uasset")); FString UexpPath = ParamsMap.FindRef(TEXT("Uexp")); const FString PackageName = ParamsMap.FindRef(TEXT("Package")); const FString ReportPath = ParamsMap.FindRef(TEXT("Report")); const bool bDryRun = Switches.Contains(TEXT("DryRun")); TArray Report; auto Line = [&Report](const FString& Text) { UE_LOG(LogLPImportLegacyStreetMap, Display, TEXT("%s"), *Text); Report.Add(Text); }; auto Finish = [&](int32 Code) { Line(FString::Printf(TEXT("CODE %d"), Code)); if (!ReportPath.IsEmpty()) { FFileHelper::SaveStringArrayToFile(Report, *ReportPath, FFileHelper::EEncodingOptions::ForceUTF8WithoutBOM); } return Code; }; if (UassetPath.IsEmpty() || (!bDryRun && PackageName.IsEmpty())) { Line(TEXT("usage : -Uasset= [-Uexp=] (-DryRun | -Package=(/Game/Heavy/StreetMap/_Pilot/ | /lpm_city_/StreetMap/)) [-Report=]")); return Finish(2); } if (UexpPath.IsEmpty()) { UexpPath = FPaths::ChangeExtension(UassetPath, TEXT("uexp")); } Line(FString::Printf(TEXT("# LPImportLegacyStreetMap %s <- %s + %s"), bDryRun ? TEXT("(DryRun)") : *PackageName, *UassetPath, *UexpPath)); // Gardes d'ecriture, avant toute lecture. if (!bDryRun) { TArray PackageParts; PackageName.ParseIntoArray(PackageParts, TEXT("/")); const bool bPackMap = PackageParts.Num() == 3 && PackageParts[0].StartsWith(TEXT("lpm_city_")) && PackageParts[1] == TEXT("StreetMap"); if ((!PackageName.StartsWith(TEXT("/Game/Heavy/StreetMap/_Pilot/")) && !bPackMap) || !FPackageName::IsValidLongPackageName(PackageName)) { Line(TEXT("ECHEC : le paquet doit etre sous /Game/Heavy/StreetMap/_Pilot/ (pilote) ou /lpm_city_/StreetMap/ (pack) ; aucun asset ecrit")); return Finish(1); } if (FPackageName::DoesPackageExist(PackageName)) { Line(FString::Printf(TEXT("ECHEC : %s existe deja ; aucun asset ecrit"), *PackageName)); return Finish(1); } } TArray Header, Body; if (!FFileHelper::LoadFileToArray(Header, *UassetPath) || !FFileHelper::LoadFileToArray(Body, *UexpPath)) { Line(TEXT("ECHEC : fichier .uasset ou .uexp illisible")); return Finish(1); } Line(FString::Printf(TEXT("sources : uasset %d octets, uexp %d octets"), Header.Num(), Body.Num())); // Resume de paquet UE4 (LegacyFileVersion -6 / -7). FContext Ctx; FReader H(Header, 0); const uint32 Tag = H.Read(); const int32 Legacy = H.Read(); if (Tag != 0x9E2A83C1u || Legacy > -6 || Legacy < -7) { Line(FString::Printf(TEXT("ECHEC : en-tete %08X legacy=%d : pas un paquet UE4 4.x pris en charge"), Tag, Legacy)); return Finish(1); } H.Read(); // LegacyUE3Version const int32 VersionUE4 = H.Read(); const int32 Licensee = H.Read(); const int32 CustomVersions = H.Read(); H.Skip(int64(FMath::Max(CustomVersions, 0)) * 20); // FGuid + int32 (format optimise) const int32 TotalHeaderSize = H.Read(); const FString FolderName = H.Str(); const uint32 PackageFlags = H.Read(); const int32 NameCount = H.Read(); const int32 NameOffset = H.Read(); H.Skip(8); // GatherableTextDataCount, GatherableTextDataOffset (paquet cuit : pas de LocalizationId) const int32 ExportCount = H.Read(); const int32 ExportOffset = H.Read(); const int32 ImportCount = H.Read(); const int32 ImportOffset = H.Read(); const bool bUnversioned = VersionUE4 == 0 && Licensee == 0; Line(FString::Printf(TEXT("en-tete : legacy=%d ue4=%d licensee=%d (%s) flags=%08X totalheader=%d noms=%d exports=%d imports=%d"), Legacy, VersionUE4, Licensee, bUnversioned ? TEXT("non versionne") : TEXT("versionne"), PackageFlags, TotalHeaderSize, NameCount, ExportCount, ImportCount)); if (H.bError || (!bUnversioned && VersionUE4 < 511) || (PackageFlags & 0x2000u) != 0) { Line(TEXT("ECHEC : resume illisible, version UE4 < 511, ou proprietes non versionnees (PKG_UnversionedProperties) : non pris en charge")); return Finish(1); } H.Pos = NameOffset; for (int32 Index = 0; Index < NameCount && !H.bError; ++Index) { Ctx.Names.Add(H.Str()); H.Skip(4); // hachages (2 x uint16) } TArray Imports; H.Pos = ImportOffset; for (int32 Index = 0; Index < ImportCount && !H.bError; ++Index) { FImport Import; Ctx.Name(H); // ClassPackage Import.ClassName = Ctx.Name(H); H.Read(); // OuterIndex Import.ObjectName = Ctx.Name(H); Imports.Add(Import); } // Export dont la classe est StreetMap. int64 SerialSize = -1, SerialOffset = -1; FString ExportName; H.Pos = ExportOffset; for (int32 Index = 0; Index < ExportCount && !H.bError; ++Index) { const int32 ClassIndex = H.Read(); H.Skip(12); // SuperIndex, TemplateIndex, OuterIndex const FString Name = Ctx.Name(H); H.Skip(4); // ObjectFlags const int64 Size = H.Read(); const int64 Offset = H.Read(); H.Skip(12 + 16 + 4 + 4 + 4 + 20); // bForcedExport, bNotForClient, bNotForServer, PackageGuid, PackageFlags, bNotAlwaysLoadedForEditorGame, bIsAsset, dependances const FString ClassName = (ClassIndex < 0 && Imports.IsValidIndex(-ClassIndex - 1)) ? Imports[-ClassIndex - 1].ObjectName : FString(); if (ClassName == TEXT("StreetMap") && SerialOffset < 0) { SerialSize = Size; SerialOffset = Offset; ExportName = Name; } } const int64 DataStart = SerialOffset - TotalHeaderSize; if (H.bError || SerialOffset < 0 || DataStart < 0 || DataStart + SerialSize > Body.Num()) { Line(TEXT("ECHEC : aucun export StreetMap lisible, ou export hors du .uexp")); return Finish(1); } Line(FString::Printf(TEXT("export : %s, %lld octets a partir de l'octet %lld du .uexp"), *ExportName, SerialSize, DataStart)); // Carte 5.8 : objet transitoire en -DryRun, sinon paquet neuf. UPackage* Package = bDryRun ? GetTransientPackage() : CreatePackage(*PackageName); const FName AssetName = bDryRun ? NAME_None : FName(*FPackageName::GetLongPackageAssetName(PackageName)); UStreetMap* StreetMap = NewObject(Package, AssetName, bDryRun ? RF_Transient : (RF_Public | RF_Standalone)); FReader R(Body, DataStart); const int64 DataEnd = DataStart + SerialSize; if (!ReadProperties(R, DataEnd, UStreetMap::StaticClass(), StreetMap, Ctx, TEXT("StreetMap"))) { Line(FString::Printf(TEXT("ECHEC : lecture des proprietes : %s ; aucun asset ecrit"), Ctx.Error.IsEmpty() ? TEXT("format inattendu") : *Ctx.Error)); return Finish(1); } Line(FString::Printf(TEXT("proprietes lues jusqu'a l'octet %lld ; reste %lld octets apres la fin de liste (attendu : 4)"), R.Pos, DataEnd - R.Pos)); for (const TPair& Pair : Ctx.Unknown) { Line(FString::Printf(TEXT("propriete inconnue en 5.8 (sautee) : %s x%d"), *Pair.Key, Pair.Value)); } for (const TPair& Pair : Ctx.Mismatch) { Line(FString::Printf(TEXT("type ou valeur incompatible (saute) : %s x%d"), *Pair.Key, Pair.Value)); } // Comptes, a comparer a Saved/MAPDL1/lire_streetmap_ue4.py --totaux. int64 RoadPoints = 0, RoadRefs = 0, BuildingPoints = 0, ParkPoints = 0, OneWay = 0; int32 Named = 0, WithAmenity = 0, WithHeight = 0; for (const FStreetMapRoad& Road : StreetMap->GetRoads()) { RoadPoints += Road.RoadPoints.Num(); OneWay += Road.bIsOneWay ? 1 : 0; } for (const FStreetMapNode& Node : StreetMap->GetNodes()) { RoadRefs += Node.RoadRefs.Num(); } for (const FStreetMapBuilding& Building : StreetMap->GetBuildings()) { BuildingPoints += Building.BuildingPoints.Num(); Named += Building.BuildingName.IsEmpty() ? 0 : 1; WithAmenity += Building.Amenity.IsEmpty() ? 0 : 1; WithHeight += Building.Height > 0.0 ? 1 : 0; } for (const FStreetMapPark& Park : StreetMap->GetParks()) { ParkPoints += Park.ParkPoints.Num(); } const FVector2D Min = StreetMap->GetBoundsMin(); const FVector2D Max = StreetMap->GetBoundsMax(); Line(FString::Printf(TEXT("TOTAUX routes=%d points_routes=%lld sens_unique=%lld noeuds=%d refs_noeuds=%lld batiments=%d points_batiments=%lld nommes=%d amenites=%d hauteurs=%d parcs=%d points_parcs=%lld eaux=%d"), StreetMap->GetRoads().Num(), RoadPoints, OneWay, StreetMap->GetNodes().Num(), RoadRefs, StreetMap->GetBuildings().Num(), BuildingPoints, Named, WithAmenity, WithHeight, StreetMap->GetParks().Num(), ParkPoints, StreetMap->GetWaters().Num())); Line(FString::Printf(TEXT("BORNES (%.1f, %.1f) a (%.1f, %.1f) cm ; soit %.2f x %.2f km"), Min.X, Min.Y, Max.X, Max.Y, (Max.X - Min.X) / 100000.0, (Max.Y - Min.Y) / 100000.0)); if (bDryRun) { Line(TEXT("DryRun : aucun asset ecrit")); return Finish(0); } FAssetRegistryModule::AssetCreated(StreetMap); FSavePackageArgs SaveArgs; SaveArgs.TopLevelFlags = RF_Public | RF_Standalone; const FString FileName = FPackageName::LongPackageNameToFilename(PackageName, FPackageName::GetAssetPackageExtension()); if (!UPackage::SavePackage(Package, StreetMap, *FileName, SaveArgs)) { Line(FString::Printf(TEXT("ECHEC : sauvegarde de %s"), *FileName)); return Finish(1); } Line(FString::Printf(TEXT("asset ecrit : %s (%s), %lld octets"), *PackageName, *FileName, IFileManager::Get().FileSize(*FileName))); return Finish(0); }