diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index e8a9df70d3..ae536fd020 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -40,7 +40,9 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.FSUtils; @@ -92,6 +94,36 @@ public Map> reportTablesMissingRegions(final List n return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA); } + public Map> reportUndeletedRegions() throws IOException { + return HBCKMetaTableAccessor.getUndeletedRegions(this.conn); + } + + public Map + removeUndeletedRegion(Map> reportMap) throws IOException { + Table table = conn.getTable(TableName.META_TABLE_NAME); + Map map = new HashMap<>(); + List list = new ArrayList<>(); + for (Map.Entry> entry : reportMap.entrySet()) { + entry.getValue().forEach(e -> list.add(new Delete(e))); + map.put(entry.getKey(), new Integer(list.size())); + table.delete(list); + list.clear(); + } + table.close(); + return map; + } + + public void deleteRegions(TableName tableName,List rows) throws IOException { + Table table = conn.getTable(tableName); + List list=new ArrayList<>(); + for (byte[] bytes : rows) { + Delete delete=new Delete(bytes); + list.add(delete); + } + table.delete(list); + table.close(); + } + List findMissingRegionsInMETA(String table) throws IOException { InternalMetaChecker missingChecker = new InternalMetaChecker<>(); return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> { diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 932f39a94a..819a33a9ea 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -101,6 +102,8 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool { private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries"; private static final String RECOVER_UNKNOWN = "recoverUnknown"; private static final String GENERATE_TABLE_INFO = "generateMissingTableDescriptorFile"; + private static final String REPORT_UNDELETED_REGIONS_IN_META = + "reportUndeletedRegionsInMeta"; private static final String FIX_META = "fixMeta"; // TODO update this map in case of the name of a method changes in Hbck interface // in org.apache.hadoop.hbase.client package. Or a new command is added and the hbck command @@ -273,6 +276,35 @@ Map> extraRegionsInMeta(String[] args) return result; } + Map> reportUndeletedRegionsInMeta(String[] args) + throws Exception { + Options options = new Options(); + Option fixOption = Option.builder("f").longOpt("fix").build(); + options.addOption(fixOption); + // Parse command-line. + CommandLineParser parser = new DefaultParser(); + CommandLine commandLine; + commandLine = parser.parse(options, args, false); + boolean fix = commandLine.hasOption(fixOption.getOpt()); + Map> result = new HashMap<>(); + try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = + new FsRegionsMetaRecoverer(this.conf)) { + Map> reportMap = + fsRegionsMetaRecoverer.reportUndeletedRegions(); + reportMap.forEach((key, value) -> + result.put(TableName.valueOf(key), + value.stream().map(Bytes::toString).collect(Collectors.toList()))); + if (fix) { + Map map = + fsRegionsMetaRecoverer.removeUndeletedRegion(reportMap); + System.out.println(formatRemovedRegionsMessage(map)); + } + } catch (IOException e) { + throw e; + } + return result; + } + private List formatNameSpaceTableParam(String... nameSpaceOrTable) { return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null; } @@ -419,6 +451,8 @@ private static String getCommandUsage() { writer.println(); usageBypass(writer); writer.println(); + usageReportUndeletedRegionsInMeta(writer); + writer.println(); usageExtraRegionsInMeta(writer); writer.println(); usageFilesystem(writer); @@ -567,6 +601,23 @@ private static void usageReplication(PrintWriter writer) { writer.println(" purge if '--fix'."); } + private static void usageReportUndeletedRegionsInMeta(PrintWriter writer){ + writer.println(" " + REPORT_UNDELETED_REGIONS_IN_META + " [OPTIONS]"); + writer.println(" Options:"); + writer.println(" -f, --fix fix meta by removing all undeleted regions found. "); + writer.println(" Reports regions present on hbase:meta, but related tables have been "); + writer.println(" deleted on hbase. Needs hbase:meta to be online. "); + writer.println(" An example triggering undeleted regions report "); + writer.println(" $ HBCK2 " + REPORT_UNDELETED_REGIONS_IN_META); + writer.println(" Returns list of undeleted regions for each not found table"); + writer.println(" If master log continues to print 'TableNotFoundException', or master "); + writer.println(" ui report RITs for those table not found regions, or hbck.jsp web page "); + writer.println(" report regions but related tables not existing, remove these undeleted "); + writer.println(" regions with '--fix' option. "); + writer.println(" You should switch active master after remove undeleted regions, then "); + writer.println(" those abnormal regions info will disappear. "); + } + private static void usageExtraRegionsInMeta(PrintWriter writer) { writer.println(" " + EXTRA_REGIONS_IN_META + " ..."); @@ -975,6 +1026,15 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc return EXIT_FAILURE; } break; + case REPORT_UNDELETED_REGIONS_IN_META: + try { + Map> report = + reportUndeletedRegionsInMeta(purgeFirst(commands)); + System.out.println(formatUndeletedRegionReport(report)); + }catch (Exception e) { + return EXIT_FAILURE; + } + break; case GENERATE_TABLE_INFO: if(commands.length != 2) { @@ -987,6 +1047,7 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc break; default: + System.out.println("REPORT_UNDELETED_REGIONS_IN_META2"); showErrorMessage("Unsupported command: " + command); return EXIT_FAILURE; } @@ -1008,6 +1069,11 @@ private String formatExtraRegionsReport(Map> report) { return formatReportMessage(message, (HashMap)report, s -> s); } + private String formatUndeletedRegionReport(Map> report) { + String message = "Regions in Meta but having no valid table, for each table:\n\t"; + return formatReportMessage(message, (HashMap)report, s -> s); + } + private String formatReportMessage(String reportMessage, Map> report, Function resolver){ final StringBuilder builder = new StringBuilder(); @@ -1067,6 +1133,16 @@ private String formatRemovedRegionsMessage(int totalRemoved, return finalText.toString(); } + private String formatRemovedRegionsMessage(Map map) { + final StringBuilder finalText = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + finalText.append("\n\tRegions that had relate to the not found table '") + .append(entry.getKey()).append("' and got removed from Meta: ") + .append(entry.getValue()).append("\n"); + } + return finalText.toString(); + } + private String buildHbck2AssignsCommand(List regions) { final StringBuilder builder = new StringBuilder(); builder.append("assigns "); diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index 8d50bb5ada..e7747c51de 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -264,6 +264,40 @@ public static List getAllRegions(Connection conn) throws IOException }); } + + /** + * List all undeleted regions currently in META. + * Undeleted regions means table not exist on hbase, but regions still in META. + * @param conn a valid, open connection. + * @return a Map of all dirty metadata in META. + * @throws IOException on any issues related with scanning meta table + */ + public static Map> getUndeletedRegions(Connection conn) + throws IOException { + final Map> undeletedTableRegions = new HashMap<>(); + final Map tableNameMap = new HashMap<>(); + List tables = getTables(conn); + tables.forEach(tableName -> tableNameMap.put(tableName.getNameAsString(), tableName)); + Table metaTable = conn.getTable(TableName.META_TABLE_NAME); + Scan scan = new Scan(); + ResultScanner resultScanner = metaTable.getScanner(scan); + for (Result result : resultScanner) { + byte[] rowBytes = result.getRow(); + String row = Bytes.toString(rowBytes); + String tableName = row.split(",")[0]; + if (!tableNameMap.containsKey(tableName)) { + if (undeletedTableRegions.containsKey(tableName)) { + undeletedTableRegions.get(tableName).add(rowBytes); + } else { + List list = new ArrayList<>(); + list.add(rowBytes); + undeletedTableRegions.put(tableName, list); + } + } + } + return undeletedTableRegions; + } + /** * Scans all "table:state" cell values existing in meta and returns as a map of * TableName as key and TableState as the value. diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 14ab7ddf25..df3964a976 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -72,10 +72,14 @@ public class TestHBCK2 { private static final Logger LOG = LoggerFactory.getLogger(TestHBCK2.class); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName()); + private static final TableName DELETED_TABLE_NAME = TableName. + valueOf(TestHBCK2.class.getSimpleName() + "-DELETED"); private static final TableName REGION_STATES_TABLE_NAME = TableName. valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES"); private final static String ASSIGNS = "assigns"; private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta"; + private static final String REPORT_UNDELETED_REGIONS_IN_META = + "reportUndeletedRegionsInMeta"; @Rule public TestName testName = new TestName(); @@ -324,6 +328,12 @@ private TableName createTestTable(int totalRegions) throws IOException { return tableName; } + private TableName createTestTable(int totalRegions, String name) throws IOException { + TableName tableName = TableName.valueOf(name); + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("family1"), totalRegions); + return tableName; + } + private void testAddMissingRegionsInMetaForTables(int missingRegions, int totalRegions) throws Exception { TableName tableName = createTestTable(totalRegions); @@ -610,4 +620,69 @@ private void testReportExtraRegionsInMeta(int extraRegionsInTestTbl, assertEquals(expectedTotalExtraRegions, resultingExtraRegions); } + private void deleteTableFamilyRegion() { + try { + HBCKMetaTableAccessor.MetaScanner scanner = + new HBCKMetaTableAccessor.MetaScanner<>(); + List rows = scanner.scanMeta(TEST_UTIL.getConnection(), + scan -> scan.addColumn(HConstants.TABLE_FAMILY,HConstants.TABLE_STATE_QUALIFIER), + r -> { + byte[] bytes = r.getRow(); + String row=Bytes.toString(bytes); + String tableName = row.split(",")[0]; + return tableName.equals(DELETED_TABLE_NAME.getNameAsString()) ? bytes : null; + }); + try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = + new FsRegionsMetaRecoverer(TEST_UTIL.getConfiguration())) { + fsRegionsMetaRecoverer.deleteRegions(TableName.META_TABLE_NAME, rows); + } + } catch (IOException e) { + fail(e.getMessage()); + } + } + + @Test + public void testReportUndeletedRegionsInMeta() throws Exception { + testReportUndeletedRegionsInMeta(5,5); + } + + private void testReportUndeletedRegionsInMeta(int undeletedRegions, + int expectedUndeletedRegions, String... namespaceOrTable) throws Exception { + createTestTable(5, DELETED_TABLE_NAME.getNameAsString()); + List regions = HBCKMetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), DELETED_TABLE_NAME); + regions.subList(0, undeletedRegions).forEach(r -> { + deleteRegionDir(DELETED_TABLE_NAME, r.getEncodedName()); + }); + deleteTableFamilyRegion(); + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + final Map> report = + hbck.reportUndeletedRegionsInMeta(namespaceOrTable); + long resultingExtraRegions = report.keySet().stream().mapToLong(nsTbl -> + report.get(nsTbl).size()).sum(); + assertEquals(expectedUndeletedRegions, resultingExtraRegions); + String expectedResult = "Regions in Meta but having no valid table, for each table:\n"; + String result = testFormatUndeleteRegionsInMeta(null); + //validates initial execute message + assertTrue(result.contains(expectedResult)); + //validates our test table region is reported as extra + expectedResult = "\t" + DELETED_TABLE_NAME.getNameAsString() + "->\n\t\t" + + DELETED_TABLE_NAME.getNameAsString(); + assertTrue(result.contains(expectedResult)); + //validates remove region with --fix + result = testFormatUndeleteRegionsInMeta("-f"); + expectedResult = "\n\tRegions that had relate to the not found table '" + + DELETED_TABLE_NAME.getNameAsString() + "' and got removed from Meta: " + + resultingExtraRegions; + assertTrue(result.contains(expectedResult)); + } + + private String testFormatUndeleteRegionsInMeta(String options) throws IOException { + if (options == null) { + return testRunWithArgs(new String[]{REPORT_UNDELETED_REGIONS_IN_META}); + } else { + return testRunWithArgs(new String[]{REPORT_UNDELETED_REGIONS_IN_META, options}); + } + } + }