AWS DMS不提供100％迁移

Question

我们所有的数据库都是从内部迁移到亚马逊极光。我们的数据库大小约为136GB，而且几个表每个都有数百万条记录。然而，在完全加载完成数百万行之后，大约200,000到300,000行被迁移。因为我们不熟悉DMS，所以我们不知道我们在哪里落下。任何人都知道如何迁移精确的行数。

migration type :full load

以下是我们的AWS DMS任务设置

{
  "TargetMetadata": {
    "TargetSchema": "",
    "SupportLobs": true,
    "FullLobMode": true,
    "LobChunkSize": 64,
    "LimitedSizeLobMode": false,
    "LobMaxSize": 0,
    "LoadMaxFileSize": 0,
    "ParallelLoadThreads": 0,
    "BatchApplyEnabled": false
  },
  "FullLoadSettings": {
    "FullLoadEnabled": true,
    "ApplyChangesEnabled": false,
    "TargetTablePrepMode": "TRUNCATE_BEFORE_LOAD",
    "CreatePkAfterFullLoad": false,
    "StopTaskCachedChangesApplied": false,
    "StopTaskCachedChangesNotApplied": false,
    "ResumeEnabled": false,
    "ResumeMinTableSize": 100000,
    "ResumeOnlyClusteredPKTables": true,
    "MaxFullLoadSubTasks": 15,
    "TransactionConsistencyTimeout": 600,
    "CommitRate": 10000
  },
  "Logging": {
    "EnableLogging": true,
    "LogComponents": [
      {
        "Id": "SOURCE_UNLOAD",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "SOURCE_CAPTURE",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "TARGET_LOAD",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "TARGET_APPLY",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      },
      {
        "Id": "TASK_MANAGER",
        "Severity": "LOGGER_SEVERITY_DEFAULT"
      }
    ],
    "CloudWatchLogGroup": "dms-tasks-krishna-smartdata",
    "CloudWatchLogStream": "dms-task-UERQWLR6AYHYIEKMR3HN2VL7T4"
  },
  "ControlTablesSettings": {
    "historyTimeslotInMinutes": 5,
    "ControlSchema": "",
    "HistoryTimeslotInMinutes": 5,
    "HistoryTableEnabled": true,
    "SuspendedTablesTableEnabled": true,
    "StatusTableEnabled": true
  },
  "StreamBufferSettings": {
    "StreamBufferCount": 3,
    "StreamBufferSizeInMB": 8,
    "CtrlStreamBufferSizeInMB": 5
  },
  "ChangeProcessingDdlHandlingPolicy": {
    "HandleSourceTableDropped": true,
    "HandleSourceTableTruncated": true,
    "HandleSourceTableAltered": true
  },
  "ErrorBehavior": {
    "DataErrorPolicy": "LOG_ERROR",
    "DataTruncationErrorPolicy": "LOG_ERROR",
    "DataErrorEscalationPolicy": "SUSPEND_TABLE",
    "DataErrorEscalationCount": 0,
    "TableErrorPolicy": "SUSPEND_TABLE",
    "TableErrorEscalationPolicy": "STOP_TASK",
    "TableErrorEscalationCount": 0,
    "RecoverableErrorCount": -1,
    "RecoverableErrorInterval": 5,
    "RecoverableErrorThrottling": true,
    "RecoverableErrorThrottlingMax": 1800,
    "ApplyErrorDeletePolicy": "IGNORE_RECORD",
    "ApplyErrorInsertPolicy": "LOG_ERROR",
    "ApplyErrorUpdatePolicy": "LOG_ERROR",
    "ApplyErrorEscalationPolicy": "LOG_ERROR",
    "ApplyErrorEscalationCount": 0,
    "FullLoadIgnoreConflicts": true
  },
  "ChangeProcessingTuning": {
    "BatchApplyPreserveTransaction": true,
    "BatchApplyTimeoutMin": 1,
    "BatchApplyTimeoutMax": 30,
    "BatchApplyMemoryLimit": 500,
    "BatchSplitSize": 0,
    "MinTransactionSize": 1000,
    "CommitTimeout": 1,
    "MemoryLimitTotal": 1024,
    "MemoryKeepTime": 60,
    "StatementCacheSize": 50
  }
}

映射方法：

{
  "rules": [
    {
      "rule-type": "selection",
      "rule-id": "1",
      "rule-name": "1",
      "object-locator": {
        "schema-name": "dbo",
        "table-name": "%"
      },
      "rule-action": "include"
    },
    {
      "rule-type": "transformation",
      "rule-id": "2",
      "rule-name": "2",
      "rule-target": "schema",
      "object-locator": {
        "schema-name": "dbo"
      },
      "rule-action": "rename",
      "value": "smartdata_int"
    }
  ]
}

Answer 1

您应该可以选择为每个DMS任务设置CloudWatch日志。你有没有检查过这个任务的日志？你有varchar / text columns> 32KB吗？将数据迁移到像redshift这样的目标时，这些将被截断，因此请注意，这将计入您的错误计数。

Answer 2

首先要做的是increase log level：

"Logging": {
    "EnableLogging": true,
    "LogComponents": [{
        "Id": "SOURCE_UNLOAD",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "SOURCE_CAPTURE",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "TARGET_LOAD",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "TARGET_APPLY",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    },{
        "Id": "TASK_MANAGER",
        "Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
    }]
  },

然后，您将能够获得有关错误发生的详细信息。

Answer 3

启用验证：

https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Validating.html

这将减慢迁移速度，因此您也可以将云分解为多个任务并在多个复制实例上运行它们，将规则1扩展为多个规则，而不是'％'添加满足表的子集的条件。

您可能还尝试使用不同的复制引擎，3.1.1刚刚发布，在撰写本文时没有3.1.1的发行说明。

https://docs.aws.amazon.com/dms/latest/userguide/CHAP_ReleaseNotes.html