diff --git a/splitting/.classpath b/splitting/.classpath new file mode 100644 index 000000000..fceb4801b --- /dev/null +++ b/splitting/.classpath @@ -0,0 +1,6 @@ + + + + + + diff --git a/splitting/.project b/splitting/.project new file mode 100644 index 000000000..255dd984a --- /dev/null +++ b/splitting/.project @@ -0,0 +1,17 @@ + + + splitting + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/splitting/.settings/org.eclipse.jdt.core.prefs b/splitting/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..3a2153707 --- /dev/null +++ b/splitting/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/splitting/README.md b/splitting/README.md new file mode 100644 index 000000000..a83febc96 --- /dev/null +++ b/splitting/README.md @@ -0,0 +1,15 @@ +# Repository Splitting + +This project provides tools for splitting the Xtext repository as described [in the Wiki](https://github.com/eclipse/xtext/wiki/Restructuring). This is the proposed work flow: + + 1. `../gradlew findProjects` creates `build/splitting/all-projects.txt`, a list of all projects (paths with a `.project` file) that have ever been in the history, and `build/splitting/unmapped-paths.txt`, a list of paths that are not covered by any project. + 2. Create `splitting.txt` based on the results of the previous step. This file must consist of lines of the form `path >> target-repos`, where `path` can be any file or directory in the repository and `target-repos` is either `delete` or a list of repository ids where that path should be included. + 3. `../gradlew clean validateSplitting` checks whether `splitting.txt` covers all files found in the history. + 4. `../gradlew generateRemovals` creates `build/splitting/removals-*.txt` for each target repository. + +Now you have all necessary data to perform the actual splitting. In order to do this: + + 1. Create and checkout a new branch. + 2. Run `../gradlew *FilterBranch`, where `*` is the target repository id. + 3. Find something else to do (on my machine the previous command takes more than one hour). + 4. Push the resulting branch to the corresponding repository. diff --git a/splitting/build.gradle b/splitting/build.gradle new file mode 100644 index 000000000..c380b2c77 --- /dev/null +++ b/splitting/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java' + +sourceSets.main.java.srcDirs = ['src'] + +def outputFilesDir = "$buildDir/splitting" +def allHistoryFiles = file(outputFilesDir + "/all-files.txt") +def allProjects = file(outputFilesDir + "/all-projects.txt") +def unmappedPaths = file(outputFilesDir + "/unmapped-paths.txt") +def splittingFile = file("splitting.txt") +def repositories = ["core", "extras", "lib", "xtend", "eclipse", "idea", "web", "maven", "xtext-website", "xtend-website"] + +task listFiles(type: Exec) { + group 'Splitting Helper' + description 'Creates a list of all files that have ever been in the git history.' + outputs.file allHistoryFiles + workingDir '..' + commandLine 'git', 'log', '--pretty=format:', '--name-only', '--diff-filter=A' + doFirst { + new File(allHistoryFiles.parent).mkdirs() + standardOutput = new FileOutputStream(allHistoryFiles) + } +} + +task findProjects(type: JavaExec) { + group 'Splitting Helper' + description 'Creates a list of all projects that have ever been in the git history.' + dependsOn(sourceSets.main.runtimeClasspath, listFiles) + inputs.file allHistoryFiles + outputs.files allProjects, unmappedPaths + classpath = sourceSets.main.runtimeClasspath.filter{it.exists()} + main = "org.eclipse.xtext.splitting.FindProjects" + args outputFilesDir +} + +task validateSplitting(type: JavaExec) { + group 'Splitting Helper' + description 'Checks splitting.txt for validity.' + dependsOn(sourceSets.main.runtimeClasspath, listFiles) + inputs.files splittingFile, allHistoryFiles + classpath = sourceSets.main.runtimeClasspath.filter{it.exists()} + main = "org.eclipse.xtext.splitting.ValidateSplitting" + args splittingFile, outputFilesDir +} + +task generateRemovals(type: JavaExec) { + group 'Splitting Helper' + description 'Generates a list of files to remove for each repository.' + dependsOn(sourceSets.main.runtimeClasspath, validateSplitting) + inputs.files splittingFile + outputs.files repositories.collect{ file("$outputFilesDir/removals-${it}.txt") } + classpath = sourceSets.main.runtimeClasspath.filter{it.exists()} + main = "org.eclipse.xtext.splitting.GenerateRemovals" + args splittingFile, outputFilesDir +} + +repositories.each { targetRepo -> + def removalsFile = file("$outputFilesDir/removals-${targetRepo}.txt") + task "${targetRepo}FilterBranch"(type: Exec) { + group 'Git History Manipulation' + description "Filters the git history for the target repository ${targetRepo}." + dependsOn(generateRemovals) + inputs.file removalsFile + workingDir '..' + commandLine 'splitting/git-filter-branch.sh', removalsFile.path, file('.').canonicalPath + } +} diff --git a/splitting/git-filter-branch.sh b/splitting/git-filter-branch.sh new file mode 100755 index 000000000..e7bd12306 --- /dev/null +++ b/splitting/git-filter-branch.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +REMOVALS=`cat $1 | tr '\n' ' '` +git filter-branch -f --prune-empty \ + --index-filter "git rm -qrf --cached --ignore-unmatch $REMOVALS" \ + --parent-filter $2/parent-filter.sh \ + HEAD diff --git a/splitting/parent-filter.sh b/splitting/parent-filter.sh new file mode 100755 index 000000000..64f90fa28 --- /dev/null +++ b/splitting/parent-filter.sh @@ -0,0 +1,10 @@ +#!/bin/bash +read INPUT +if [ -n "$INPUT" ] +then + SHOW_BRANCH=`git show-branch --independent ${INPUT//-p/}` + for PARENT in $SHOW_BRANCH + do + echo -n "-p $PARENT " + done +fi diff --git a/splitting/settings.gradle b/splitting/settings.gradle new file mode 100644 index 000000000..e69de29bb diff --git a/splitting/splitting.txt b/splitting/splitting.txt new file mode 100644 index 000000000..87269e6b4 --- /dev/null +++ b/splitting/splitting.txt @@ -0,0 +1,255 @@ +plugins/org.eclipse.xtext >> core +plugins/org.eclipse.xtext.core >> core +plugins/org.eclipse.xtext.ide >> core +plugins/org.eclipse.xtext.util >> core +plugins/org.eclipse.xtext.xtext.generator >> core +plugins/org.eclipse.xtext.xtext.wizard >> core +tests/org.eclipse.xtext.tests >> core, extras +tests/org.eclipse.xtext.ide.tests >> core +devtools/org.eclipse.xtext.bootstrap >> core +devtools/org.eclipse.xtext.contributor >> core + +devtools/org.eclipse.xtext.language-generator >> extras +plugins/org.eclipse.xtext.builder.standalone >> extras +plugins/org.eclipse.xtext.common.types >> extras +plugins/org.eclipse.xtext.ecore >> extras +plugins/org.eclipse.xtext.generator >> extras +plugins/org.eclipse.xtext.generator.parser >> extras +plugins/org.eclipse.xtext.index >> extras +plugins/org.eclipse.xtext.java >> extras +plugins/org.eclipse.xtext.purexbase >> extras +plugins/org.eclipse.xtext.purexbase.test >> extras +plugins/org.eclipse.xtext.smap >> extras +plugins/org.eclipse.xtext.xbase >> extras +plugins/org.eclipse.xtext.xbase.ide >> extras +tests/org.eclipse.xtext.generator.tests >> extras +tests/org.eclipse.xtext.common.types.test >> extras, eclipse +tests/org.eclipse.xtext.common.types.tests >> extras, eclipse +tests/org.eclipse.xtext.java.tests >> extras, eclipse +tests/org.eclipse.xtext.purexbase.tests >> extras, eclipse +tests/org.eclipse.xtext.testlanguages >> extras +tests/org.eclipse.xtext.testlanguages.ide >> extras +tests/org.eclipse.xtext.testlanguages.tests >> extras +tests/org.eclipse.xtext.xbase.tests >> extras + +maven/org.eclipse.xtend.lib.gwt.test >> lib +maven/org.eclipse.xtext.xbase.lib.slim >> lib +plugins/org.eclipse.xtend.lib >> lib +plugins/org.eclipse.xtend.lib.gwt >> lib +plugins/org.eclipse.xtend.lib.macro >> lib +plugins/org.eclipse.xtend.lib.maven >> lib +plugins/org.eclipse.xtend2.lib >> lib +plugins/org.eclipse.xtext.xbase.lib >> lib +plugins/org.eclipse.xtext.xbase.lib.gwt >> lib +plugins/org.eclipse.xtext.xbase.lib.maven >> lib +plugins/org.eclipse.xtext.xtend2.lib >> lib + +docs/org.eclipse.xtend.bootstrapdoc >> xtend +docs/org.eclipse.xtend.doc >> xtend +docs/org.eclipse.xtend.doc.xdoc >> xtend +examples/org.eclipse.xtend.examples >> xtend +examples/org.eclipse.xtend.examples-container >> xtend +plugins/org.eclipse.xtend.bootstrapdoc >> xtend +plugins/org.eclipse.xtend.core >> xtend +plugins/org.eclipse.xtend.doc >> xtend +plugins/org.eclipse.xtend.doc.xdoc >> xtend +plugins/org.eclipse.xtend.gradle.plugin >> xtend +plugins/org.eclipse.xtend.ide >> xtend +plugins/org.eclipse.xtend.ide.common >> xtend +plugins/org.eclipse.xtend.m2e >> xtend +plugins/org.eclipse.xtend.macro >> xtend +plugins/org.eclipse.xtend.macro.lang >> xtend +plugins/org.eclipse.xtend.macro.lang.tests >> xtend +plugins/org.eclipse.xtend.macro.lang.ui >> xtend +plugins/org.eclipse.xtend.maven >> xtend +plugins/org.eclipse.xtend.maven.android.archetype >> xtend +plugins/org.eclipse.xtend.maven.archetype >> xtend +plugins/org.eclipse.xtend.standalone >> xtend +plugins/org.eclipse.xtend.xdoc >> xtend +plugins/org.eclipse.xtext.xtend >> xtend +plugins/org.eclipse.xtext.xtend2 >> xtend +plugins/org.eclipse.xtext.xtend2.bootstrapdoc >> xtend +plugins/org.eclipse.xtext.xtend2.doc >> xtend +plugins/org.eclipse.xtext.xtend2.m2e >> xtend +plugins/org.eclipse.xtext.xtend2.ui >> xtend +tests/org.eclipse.xtend.caliper.tests >> xtend +tests/org.eclipse.xtend.core.tests >> xtend +tests/org.eclipse.xtend.core.tests.java8 >> xtend +tests/org.eclipse.xtend.ide.swtbot.tests >> xtend +tests/org.eclipse.xtend.ide.tests >> xtend +tests/org.eclipse.xtend.ide.tests.data >> xtend +tests/org.eclipse.xtend.macro.lang.tests >> xtend +tests/org.eclipse.xtend.performance.tests >> xtend +tests/org.eclipse.xtend.standalone.tests >> xtend +tests/org.eclipse.xtend.swtbot >> xtend +tests/org.eclipse.xtext.xtend.tests >> xtend +tests/org.eclipse.xtext.xtend2.standalone.tests >> xtend +tests/org.eclipse.xtext.xtend2.tests >> xtend +tests/org.eclipse.xtext.xtend2.ui.tests >> xtend + +features >> eclipse +releng >> eclipse +docs/gen_toc.jar >> eclipse +docs/org.eclipse.xtext.doc >> eclipse +docs/org.eclipse.xtext.doc.xdoc >> eclipse +examples/org.eclipse.xtext.example.domainmodel >> eclipse +examples/org.eclipse.xtext.example.domainmodel.ui >> eclipse +examples/org.eclipse.xtext.example.ecoredsl >> eclipse +examples/org.eclipse.xtext.example.ecoredsl.ui >> eclipse +examples/org.eclipse.xtext.example.fowlerdsl >> eclipse +examples/org.eclipse.xtext.example.fowlerdsl.ui >> eclipse +examples/org.eclipse.xtext.xtext.ui.examples >> eclipse +plugins/org.eclipse.xtext.activities >> eclipse +plugins/org.eclipse.xtext.builder >> eclipse +plugins/org.eclipse.xtext.common.types.edit >> eclipse +plugins/org.eclipse.xtext.common.types.shared >> eclipse +plugins/org.eclipse.xtext.common.types.shared.jdt38 >> eclipse +plugins/org.eclipse.xtext.common.types.ui >> eclipse +plugins/org.eclipse.xtext.common.ui >> eclipse +plugins/org.eclipse.xtext.compatibility.galileo >> eclipse +plugins/org.eclipse.xtext.doc >> eclipse +plugins/org.eclipse.xtext.doc.xdoc >> eclipse +plugins/org.eclipse.xtext.gmf.glue >> eclipse +plugins/org.eclipse.xtext.graphviewer.ui >> eclipse +plugins/org.eclipse.xtext.junit >> eclipse +plugins/org.eclipse.xtext.junit4 >> eclipse +plugins/org.eclipse.xtext.log4j >> eclipse +plugins/org.eclipse.xtext.logging >> eclipse +plugins/org.eclipse.xtext.m2e >> eclipse +plugins/org.eclipse.xtext.purexbase.ui >> eclipse +plugins/org.eclipse.xtext.preference >> eclipse +plugins/org.eclipse.xtext.service >> eclipse +plugins/org.eclipse.xtext.service.ui >> eclipse +plugins/org.eclipse.xtext.ui >> eclipse +plugins/org.eclipse.xtext.ui.builtin >> eclipse +plugins/org.eclipse.xtext.ui.codetemplates >> eclipse +plugins/org.eclipse.xtext.ui.codetemplates.ui >> eclipse +plugins/org.eclipse.xtext.ui.common >> eclipse +plugins/org.eclipse.xtext.ui.common.xtend >> eclipse +plugins/org.eclipse.xtext.ui.core >> eclipse +plugins/org.eclipse.xtext.ui.ecore >> eclipse +plugins/org.eclipse.xtext.ui.generator >> eclipse +plugins/org.eclipse.xtext.ui.junit >> eclipse +plugins/org.eclipse.xtext.ui.shared >> eclipse +plugins/org.eclipse.xtext.wikitext.core >> eclipse +plugins/org.eclipse.xtext.wizard >> eclipse +plugins/org.eclipse.xtext.xbase.doc >> eclipse +plugins/org.eclipse.xtext.xbase.junit >> eclipse +plugins/org.eclipse.xtext.xbase.ui >> eclipse +plugins/org.eclipse.xtext.xtext.ui >> eclipse +plugins/org.eclipse.xtext.xtext.ui.graph >> eclipse +tests/org.eclipse.xtext.automated.tests >> eclipse +tests/org.eclipse.xtext.builder.tests >> eclipse +tests/org.eclipse.xtext.example.domainmodel.ui.tests >> eclipse +tests/org.eclipse.xtext.index.tests >> eclipse +tests/org.eclipse.xtext.junit4.tests >> eclipse +tests/org.eclipse.xtext.preference.tests >> eclipse +tests/org.eclipse.xtext.service.tests >> eclipse +tests/org.eclipse.xtext.service.ui.tests >> eclipse +tests/org.eclipse.xtext.testlanguages.ui >> eclipse +tests/org.eclipse.xtext.ui.codetemplates.tests >> eclipse +tests/org.eclipse.xtext.ui.common.tests >> eclipse +tests/org.eclipse.xtext.ui.core.tests >> eclipse +tests/org.eclipse.xtext.ui.integration.tests >> eclipse +tests/org.eclipse.xtext.ui.tests >> eclipse +tests/org.eclipse.xtext.wikitext.tests >> eclipse +tests/org.eclipse.xtext.xbase.testlanguages >> eclipse +tests/org.eclipse.xtext.xbase.testlanguages.ui >> eclipse +tests/org.eclipse.xtext.xbase.ui.tests >> eclipse +tests/org.eclipse.xtext.xtext.ui.graph.tests >> eclipse +tests/org.eclipse.xtext.xtext.ui.tests >> eclipse + +intellij >> idea + +web >> web + +maven/org.eclipse.xtend.core >> maven +maven/org.eclipse.xtend.ide.common >> maven +maven/org.eclipse.xtend.maven.android.archetype >> maven +maven/org.eclipse.xtend.maven.archetype >> maven +maven/org.eclipse.xtend.maven.plugin >> maven +maven/org.eclipse.xtext >> maven +maven/org.eclipse.xtext.common.types >> maven +maven/org.eclipse.xtext.dependencies >> maven +maven/org.eclipse.xtext.ide >> maven +maven/org.eclipse.xtext.maven.plugin >> maven +maven/org.eclipse.xtext.parent >> maven +maven/org.eclipse.xtext.standalone.parent >> maven +maven/org.eclipse.xtext.target >> maven +maven/org.eclipse.xtext.tycho.parent >> maven +maven/org.eclipse.xtext.tycho.tests.parent >> maven +maven/org.eclipse.xtext.xbase >> maven +maven/org.eclipse.xtext.xtext >> maven +plugins/org.eclipse.xtend.maven.plugin >> maven +plugins/org.eclipse.xtend.standalone.maven >> maven +plugins/org.eclipse.xtend2.lib.maven >> maven +plugins/org.eclipse.xtend2.maven.plugin >> maven +plugins/org.eclipse.xtend2.standalone.maven >> maven +plugins/org.eclipse.xtext.maven.plugin >> maven +plugins/org.eclipse.xtext.standalone.maven >> maven +plugins/org.eclipse.xtext.xtend2.maven >> maven +plugins/org.eclipse.xtext.xtend2.standalone >> maven +plugins/org.eclipse.xtext.xtend2.standalone.maven >> maven + +website >> xtext-website +xtext-website >> xtext-website +xtext.website.generator >> xtext-website + +website-xtend >> xtend-website +xtend-website >> xtend-website +xtend.website.generator >> xtend-website + +.DS_Store >> delete +.gitignore >> delete +.project >> delete +.settings >> delete +CONTRIBUTING >> delete +README.md >> delete +build.gradle >> delete +develop >> delete +devtools/com.rcpquickstart.bundletestcollector >> delete +devtools/org.eclipse.xtext.reference >> delete +devtools/org.eclipse.xtext.reference.ui >> delete +devtools/org.eclipse.xtext.testcollector >> delete +docs/images >> delete +docs/org.xtext.sevenlanguages.doc.xdoc >> delete +dummy.txt >> delete +gradle >> delete +gradle-app.setting >> delete +gradle.ant >> delete +gradle.properties >> delete +gradlew >> delete +gradlew.bat >> delete +hover.patch.txt >> delete +jenkins >> delete +maven/.settings >> delete +maven/build.gradle >> delete +maven/mvnw >> delete +maven/mvnw.bat >> delete +maven/wrapper >> delete +org.eclipse.core.resources.prefs >> delete +org.eclipse.core.runtime.prefs >> delete +org.eclipse.jdt.core.prefs >> delete +org.eclipse.jdt.launching.prefs >> delete +org.eclipse.jdt.ui.prefs >> delete +org.eclipse.xtend.core.Xtend.prefs >> delete +plugins/.settings >> delete +plugins/build.gradle >> delete +plugins/com.google.collect >> delete +plugins/com.google.guava >> delete +plugins/com.google.guice >> delete +plugins/com.google.inject >> delete +plugins/images >> delete +plugins/org.antlr.gen >> delete +plugins/org.antlr.runtime >> delete +plugins/org.apache.log4j >> delete +plugins/org.eclipse.emf.index >> delete +plugins/org.eclipse.emf.index.ui >> delete +plugins/org.eclipse.xtext.doc.manual >> delete +settings.gradle >> delete +splitting >> delete +test.txt >> delete +tests/.settings >> delete +tests/build.gradle >> delete +tests/org.eclipse.emf.index.tests >> delete +tests/org.eclipse.emf.index.ui.tests >> delete diff --git a/splitting/src/org/eclipse/xtext/splitting/FindProjects.java b/splitting/src/org/eclipse/xtext/splitting/FindProjects.java new file mode 100644 index 000000000..e685c5fe7 --- /dev/null +++ b/splitting/src/org/eclipse/xtext/splitting/FindProjects.java @@ -0,0 +1,168 @@ +/******************************************************************************* + * Copyright (c) 2016 TypeFox GmbH (http://www.typefox.io) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.xtext.splitting; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class FindProjects { + + public static final String ALL_FILES = "all-files.txt"; + public static final String ALL_PROJECTS = "all-projects.txt"; + public static final String UNMAPPED_PATHS = "unmapped-paths.txt"; + + public static void main(String[] args) { + if (args.length != 1) { + fail("Expected path to output files as argument."); + } + String outputDir = args[0]; + try { + final Directory root = new Directory(".", null); + try (BufferedReader reader = new BufferedReader(new FileReader(outputDir + "/" + ALL_FILES))) { + // Gather all paths in a directory structure + String line; + while ((line = reader.readLine()) != null) { + String[] segments = line.split("/"); + DirectoryEntry current = root; + for (int i = 0; i < segments.length; i++) { + String segment = segments[i].replaceAll("\"|\\\\.", ""); + if (!segment.isEmpty() && current instanceof Directory) { + Directory currentDir = (Directory) current; + if (currentDir.entries.containsKey(segment)) + current = currentDir.entries.get(segment); + else { + DirectoryEntry newEntry; + if (i == segments.length - 1) + newEntry = new DirectoryEntry(segment, currentDir); + else + newEntry = new Directory(segment, currentDir); + currentDir.entries.put(segment, newEntry); + current = newEntry; + } + } + } + } + } + + // Find the projects and unmapped paths that are relevant for the repository history + final List projects = new ArrayList<>(); + final List otherPaths = new ArrayList<>(); + findProjects(root, projects, otherPaths, false); + + // Write the collected projects into a file + Collections.sort(projects); + try (FileWriter writer = new FileWriter(outputDir + "/" + ALL_PROJECTS)) { + for (Directory project : projects) { + writer.write(project.toString()); + writer.write('\n'); + } + } + + // Write the unmapped paths into another file + Collections.sort(otherPaths); + try (FileWriter writer = new FileWriter(outputDir + "/" + UNMAPPED_PATHS)) { + for (DirectoryEntry path : otherPaths) { + writer.write(path.toString()); + writer.write('\n'); + } + } + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + private static void fail(String message) { + System.err.print("ERROR: "); + System.err.println(message); + System.exit(1); + } + + private static boolean findProjects(Directory dir, List projects, List otherPaths, boolean skipOtherPaths) { + boolean containsProject = dir.parent != null && dir.entries.values().stream().anyMatch((e) -> e.name.equals(".project")); + if (containsProject) { + projects.add(dir); + skipOtherPaths = true; + } + Set subDirsWithProjects = new HashSet<>(); + for (DirectoryEntry entry : dir.entries.values()) { + if (entry instanceof Directory) { + Directory subDir = (Directory) entry; + if (findProjects(subDir, projects, otherPaths, skipOtherPaths)) + subDirsWithProjects.add(subDir); + } + } + if (!skipOtherPaths && !subDirsWithProjects.isEmpty()) { + for (DirectoryEntry entry : dir.entries.values()) { + if (!subDirsWithProjects.contains(entry)) + otherPaths.add(entry); + } + } + return containsProject || !subDirsWithProjects.isEmpty(); + } + + private static class DirectoryEntry implements Comparable { + final String name; + final Directory parent; + + DirectoryEntry(String name, Directory parent) { + this.name = name; + this.parent = parent; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Directory) { + Directory other = (Directory) obj; + return this.parent == other.parent && this.name.equals(other.name); + } + return false; + } + + @Override + public int hashCode() { + if (parent == null) + return name.hashCode(); + else + return parent.hashCode() * 7 + name.hashCode(); + } + + @Override + public int compareTo(DirectoryEntry other) { + if (this.parent == other.parent) + return this.name.compareTo(other.name); + else + return this.toString().compareTo(other.toString()); + } + + @Override + public String toString() { + if (parent == null || parent.parent == null) + return name; + else + return parent.toString() + "/" + name; + } + } + + private static class Directory extends DirectoryEntry { + final Map entries = new HashMap<>(); + + Directory(String name, Directory parent) { + super(name, parent); + } + } + +} diff --git a/splitting/src/org/eclipse/xtext/splitting/GenerateRemovals.java b/splitting/src/org/eclipse/xtext/splitting/GenerateRemovals.java new file mode 100644 index 000000000..ab4651dfb --- /dev/null +++ b/splitting/src/org/eclipse/xtext/splitting/GenerateRemovals.java @@ -0,0 +1,111 @@ +/******************************************************************************* + * Copyright (c) 2016 TypeFox GmbH (http://www.typefox.io) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.xtext.splitting; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.FileWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class GenerateRemovals { + + public static final Set GEN_DIRS = Collections.unmodifiableSet(new LinkedHashSet<>(Arrays.asList( + "src-gen", "xtend-gen", "xtext-gen", "emf-gen" + ))); + + public static void main(String[] args) { + if (args.length != 2) { + fail("Expected paths to splitting.txt and output files as arguments."); + } + String outputDir = args[1]; + try { + final Map> removalPaths = new HashMap<>(); + for (String targetRepo : ValidateSplitting.REPOSITORIES) { + removalPaths.put(targetRepo, new LinkedHashSet<>()); + } + + // Gather paths to be removed from the history for each target repository + try (BufferedReader reader = new BufferedReader(new FileReader(args[0]))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + String[] parts = line.split(">>"); + String[] repos = parts[1].split(","); + for (String targetRepo : ValidateSplitting.REPOSITORIES) { + boolean isInTargetRepo = false; + for (String repo : repos) { + if (targetRepo.equals(repo.trim())) { + isInTargetRepo = true; + break; + } + } + if (!isInTargetRepo) { + String path = parts[0].trim(); + removalPaths.get(targetRepo).add(path); + } + } + } + } + } + + // Add directories with generated code to the removal list + final Pattern segmentPattern = Pattern.compile("/"); + try (BufferedReader reader = new BufferedReader(new FileReader(outputDir + "/" + FindProjects.ALL_FILES))) { + String line; + while ((line = reader.readLine()) != null) { + String file = line.replaceAll("\"|\\\\.", ""); + for (String genDir : GEN_DIRS) { + int genDirIndex = file.indexOf(genDir); + if (genDirIndex > 0) { + for (String targetRepo : ValidateSplitting.REPOSITORIES) { + Set repoRemovals = removalPaths.get(targetRepo); + Matcher matcher = segmentPattern.matcher(file); + boolean foundRemoval = false; + int genDirEndIndex = genDirIndex + genDir.length(); + while (!foundRemoval && matcher.find() && matcher.start() <= genDirEndIndex) { + if (repoRemovals.contains(file.substring(0, matcher.start()))) + foundRemoval = true; + } + if (!foundRemoval) + repoRemovals.add(file.substring(0, genDirEndIndex)); + } + } + } + } + } + + // Write a removal list for each target repository + for (String targetRepo : ValidateSplitting.REPOSITORIES) { + try (FileWriter writer = new FileWriter(outputDir + "/removals-" + targetRepo + ".txt")) { + for (String path : removalPaths.get(targetRepo)) { + writer.write(path); + writer.write('\n'); + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + private static void fail(String message) { + System.err.print("ERROR: "); + System.err.println(message); + System.exit(1); + } + +} diff --git a/splitting/src/org/eclipse/xtext/splitting/ValidateSplitting.java b/splitting/src/org/eclipse/xtext/splitting/ValidateSplitting.java new file mode 100644 index 000000000..ab6e66c2d --- /dev/null +++ b/splitting/src/org/eclipse/xtext/splitting/ValidateSplitting.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * Copyright (c) 2016 TypeFox GmbH (http://www.typefox.io) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.xtext.splitting; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ValidateSplitting { + + public static final Set REPOSITORIES = Collections.unmodifiableSet(new LinkedHashSet<>(Arrays.asList( + "core", "extras", "lib", "xtend", "eclipse", "idea", "web", "maven", "xtext-website", "xtend-website" + ))); + + public static final String DELETE = "delete"; + + public static void main(String[] args) { + if (args.length != 2) { + fail("Expected paths to splitting.txt and output files as arguments."); + } + String outputDir = args[1]; + try { + + // Validate repositories and gather all paths from the splitting file + final Set specifiedPaths = new HashSet<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(args[0]))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + String[] parts = line.split(">>"); + if (parts.length != 2) + fail("Invalid line: " + line); + if (!DELETE.equals(parts[1].trim())) { + String[] repos = parts[1].split(","); + if (repos.length == 0) + fail("Invalid line: " + line); + for (String repo : repos) { + String trimmed = repo.trim(); + if (!REPOSITORIES.contains(trimmed)) + fail("Invalid repository: " + trimmed); + } + } + String path = parts[0].trim(); + specifiedPaths.add(path); + } + } + } + + // Check whether each file has a specified path as prefix + final Pattern segmentPattern = Pattern.compile("/"); + try (BufferedReader reader = new BufferedReader(new FileReader(outputDir + "/" + FindProjects.ALL_FILES))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + String file = line.replaceAll("\"|\\\\.", ""); + if (!specifiedPaths.contains(file)) { + Matcher matcher = segmentPattern.matcher(file); + boolean foundSplitting = false; + while (!foundSplitting && matcher.find()) { + if (specifiedPaths.contains(file.substring(0, matcher.start()))) + foundSplitting = true; + } + if (!foundSplitting) + fail("File not covered by splitting: " + file); + } + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + private static void fail(String message) { + System.err.print("ERROR: "); + System.err.println(message); + System.exit(1); + } + +}