Mysql 集群插入停止 TCROLLBACKREQ

标签 mysql bulkinsert mysql-cluster

我有一个带有 4 个数据节点的 mysql 集群。我正在尝试插入 800 万行，使用加载数据文件中的 50k block 来保持事务较小。就像发条一样，每向表中插入 100 万行后，插入就会暂停近 20 分钟，然后再重新开始。我也在日志中收到此错误。

2014-07-17 22:34:59 [NdbApi] 错误 -- 发送带有错误标志的 TCROLLBACKREQ

我假设我需要更改配置

[ndbd default]
# Options affecting ndbd processes on all data nodes:
NoOfReplicas=2    # Number of replicas
DataMemory=90G    # How much memory to allocate for data storage
IndexMemory=10G   # How much memory to allocate for index storage
                  # For DataMemory and IndexMemory, we have used the
                  # default values. Since the "world" database takes up
                  # only about 500KB, this should be more than enough for
                  # this example Cluster setup.
NoOfFragmentLogFiles=20
MaxNoOfOrderedIndexes=1000
MaxNoOfAttributes=10000
MaxBufferedEpochs=10000
MaxNoOfConcurrentTransactions=1000000
MaxNoOfConcurrentOperations=10000000
MaxNoOfLocalOperations=11000000

[tcp default]
# TCP/IP options:
#portnumber=2202   # This the default; however, you can use any
                  # port that is free for all the hosts in the cluster
                  # Note: It is recommended that you do not specify the port
                  # number at all and simply allow the default value to be used
                  # instead

[ndb_mgmd]
# Management process options:
hostname=########.173          # Hostname or IP address of MGM node
datadir=/var/lib/mysql-cluster  # Directory for MGM node log files
nodeId=1

[ndbd]
# Options for data node "A":
hostname=########.173          # Hostname or IP address
datadir=/usr/local/mysql/data   # Directory for this data node's data files
ServerPort=50501
nodeId=2
MaxNoOfExecutionThreads=4

[ndbd]
# Options for data node "B":
hostname=########.174          # Hostname or IP address
datadir=/usr/local/mysql/data   # Directory for this data node's data files
ServerPort=50502
nodeId=3
MaxNoOfExecutionThreads=4

[ndbd]
# Options for data node "C":
hostname=########.175          # Hostname or IP address
datadir=/usr/local/mysql/data   # Directory for this data node's data files
ServerPort=50503
nodeId=5
MaxNoOfExecutionThreads=4

[ndbd]
# Options for data node "D":
hostname=########.176          # Hostname or IP address
datadir=/usr/local/mysql/data   # Directory for this data node's data files
ServerPort=50504
nodeId=6
MaxNoOfExecutionThreads=4

#[ndbd]
# Options for data node "E":
#hostname=########.177          # Hostname or IP address
#datadir=/usr/local/mysql/data   # Directory for this data node's data files
#ServerPort=50505
#nodeId=7
#MaxNoOfExecutionThreads=4

[mysqld]
# SQL node options:
nodeId=4
hostname=########.174          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=7
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=8
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=9
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=10
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=11
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=12
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=13
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

[mysqld]
# SQL node options:
nodeId=14
hostname=########.177          # Hostname or IP address
                                # (additional mysqld connections can be
                                # specified for this node for various
                                # purposes such as running ndb_restore)

最佳答案

将我的配置更改为这个解决了问题

[TCP DEFAULT]
SendBufferMemory=8M
ReceiveBufferMemory=8M

[NDB_MGMD DEFAULT]
PortNumber=1186
Datadir=/data/mysqlcluster/

[NDB_MGMD]
NodeId=1
Hostname=vacnode6.mc.vanderbilt.edu
LogDestination=FILE:filename=ndb_1_cluster.log,maxsize=10000000,maxfiles=6
ArbitrationRank=1

[NDBD DEFAULT]
NoOfReplicas=2
ServerPort=2200
Datadir=/data/mysqlcluster/
FileSystemPathDD=/data/mysqlcluster/
BackupDataDir=/data/mysqlcluster//backup/
#FileSystemPathUndoFiles=/data/mysqlcluster/
#FileSystemPathDataFiles=/data/mysqlcluster/
DataMemory=95695M
IndexMemory=11962M
LockPagesInMainMemory=1

MaxNoOfConcurrentOperations=100000
MaxNoOfConcurrentTransactions=16384

StringMemory=25
MaxNoOfTables=4096
MaxNoOfOrderedIndexes=2048
MaxNoOfUniqueHashIndexes=512
MaxNoOfAttributes=24576
MaxNoOfTriggers=14336

### Params for REDO LOG 
FragmentLogFileSize=256M
InitFragmentLogFiles=SPARSE
NoOfFragmentLogFiles=374
RedoBuffer=64M

TransactionBufferMemory=8M

TimeBetweenGlobalCheckpoints=1000
TimeBetweenEpochs=100

TimeBetweenEpochsTimeout=0

### Params for LCP 
DiskCheckpointSpeedInRestart=100M
DiskCheckpointSpeed=10M
TimeBetweenLocalCheckpoints=20

### Heartbeating 
HeartbeatIntervalDbDb=15000
HeartbeatIntervalDbApi=15000

### Params for setting logging 
MemReportFrequency=30
BackupReportFrequency=10
LogLevelStartup=15
LogLevelShutdown=15
LogLevelCheckpoint=8
LogLevelNodeRestart=15

### Params for BACKUP 
BackupMaxWriteSize=1M
BackupDataBufferSize=24M
BackupLogBufferSize=16M
BackupMemory=40M

### Params for ODIRECT 
#Reports indicates that odirect=1 can cause io errors (os err code 5) on some systems. You must test.
#ODirect=1

### Watchdog 
TimeBetweenWatchdogCheckInitial=60000

### TransactionInactiveTimeout  - should be enabled in Production 
TransactionInactiveTimeout=60000
### New 7.1.10 redo logging parameters 
RedoOverCommitCounter=3
RedoOverCommitLimit=20
### REALTIME EXTENSIONS 
#RealTimeScheduler=1
### REALTIME EXTENSIONS FOR 6.3 ONLY
#SchedulerExecutionTimer=80
#SchedulerSpinTimer=40

### DISK DATA 
SharedGlobalMemory=20M
DiskPageBufferMemory=64M

### Multithreading 
MaxNoOfExecutionThreads=8
NoOfFragmentLogParts=4

### Increasing the LongMessageBuffer b/c of a bug (20090903)
LongMessageBuffer=32M

BatchSizePerLocalScan=512
[NDBD]
NodeId=2
Hostname=vacnode1.mc.vanderbilt.edu

### REALTIME ASPECTS - THREAD BINDING 
### READ http://johanandersson.blogspot.com/2008/02/mysql-cluster-features-what-they-are.html#realtime
#LockExecuteThreadToCPU=X
#LockMaintThreadsToCPU=Y

[NDBD]
NodeId=3
Hostname=vacnode2.mc.vanderbilt.edu

### REALTIME ASPECTS - THREAD BINDING 
### READ http://johanandersson.blogspot.com/2008/02/mysql-cluster-features-what-they-are.html#realtime
#LockExecuteThreadToCPU=X
#LockMaintThreadsToCPU=Y

[NDBD]
NodeId=4
Hostname=vacnode3.mc.vanderbilt.edu

### REALTIME ASPECTS - THREAD BINDING 
### READ http://johanandersson.blogspot.com/2008/02/mysql-cluster-features-what-they-are.html#realtime
#LockExecuteThreadToCPU=X
#LockMaintThreadsToCPU=Y

[NDBD]
NodeId=5
Hostname=vacnode4.mc.vanderbilt.edu

### REALTIME ASPECTS - THREAD BINDING 
### READ http://johanandersson.blogspot.com/2008/02/mysql-cluster-features-what-they-are.html#realtime
#LockExecuteThreadToCPU=X
#LockMaintThreadsToCPU=Y

[MYSQLD DEFAULT]
DefaultOperationRedoProblemAction=QUEUE
BatchSize=512
#BatchByteSize=2048K
#MaxScanBatchSize=2048K

[MYSQLD]
NodeId=6
[MYSQLD]
NodeId=7
[MYSQLD]
NodeId=8
[MYSQLD]
NodeId=9

[MYSQLD]
NodeId=10
[MYSQLD]
NodeId=11
[MYSQLD]
NodeId=12
[MYSQLD]
NodeId=13

[MYSQLD]
NodeId=14
[MYSQLD]
NodeId=15
[MYSQLD]
NodeId=16
[MYSQLD]
NodeId=17

[MYSQLD]
NodeId=18
[MYSQLD]
NodeId=19
[MYSQLD]
NodeId=20
[MYSQLD]
NodeId=21

[MYSQLD]
NodeId=22
[MYSQLD]
NodeId=23
[MYSQLD]
NodeId=24
[MYSQLD]
NodeId=25

[MYSQLD]
NodeId=26
[MYSQLD]
NodeId=27
[MYSQLD]
NodeId=28
[MYSQLD]
NodeId=29

[MYSQLD]
NodeId=30
[MYSQLD]
NodeId=31
[MYSQLD]
NodeId=32
[MYSQLD]
NodeId=33

[MYSQLD]
NodeId=34
[MYSQLD]
NodeId=35
[MYSQLD]
NodeId=36
[MYSQLD]
NodeId=37

[MYSQLD]
[MYSQLD]
[MYSQLD]
[MYSQLD]
### SLOTS (one for each ndb_mgmd) FOR HELPER APPLICATIONS SUCH AS ndb_show_tables etc
[MYSQLD]
Hostname=vacnode6.mc.vanderbilt.edu
### SLOT FOR CMON
[MYSQLD]
Hostname=something.mc.vanderbilt.edu

关于Mysql 集群插入停止 TCROLLBACKREQ，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/24816873/

上一篇：mysql - 如果查询返回空集如何获取 0

下一篇：php - 将查询数据保存到数组

相关文章：

php - 将字段数据与另一个表进行比较

postgresql - 批量插入亿条记录

mysql - 我无法在 ubuntu 上使用 ndb 集群引擎创建表

mysql - 是否有可能，使用 mysql 的分散解决方案

MySQL - 如果 ID 和时间戳在时间范围内唯一的组合约束，则防止插入记录

php - 将csv文件上传并导入到mysql中的多个表中

oracle - 批量收集到对象表中

php - 如何处理包含 500 多个项目的 Symfony 表单集合

python - 从 python 应用程序连接到 mysql 集群

mysql - 如何使用缺失行的默认值进行 mysql 连接

©2024 IT工具网  联系我们