OrientDB Server 阻塞问题分析(一)

作者: jiaxiaolei | 来源:发表于2018-03-22 17:05 被阅读104次

在OrientDB pyorient 的使用中发现，如果因为执行非法的语句（比如 select * from xxxxxxxx, xxxxxxxx 是一个不存在的class）导致OrientDB Server 解析失败（com.orientechnologies.orient.core.exception.OQueryParsingException: Error on parsing query), pyorient 会抛出 MemoryError。此时pyorient client 就不能正常使用，会“阻塞”当前进程，需要重新初始化client 才可以继续使用。

具体示例如下：

# filename: test.py 

import logging

from tornado.options import parse_command_line
import pyorient

cmdb_port = 2424
cmdb_url = "db-vip"
db_username = "root"
db_password = "root"
db_name = "cmdb_test"

client = pyorient.OrientDB(cmdb_url, cmdb_port)
session_id = client.connect(db_username, db_password)



def test_cmd(sql_cmd):
    logging.info('come into test_cmd')
    try:
        client.db_open(db_name, db_username, db_password)
        ret = client.command(sql_cmd)
        logging.info('ret len: %s', len(ret))
    except Exception as e:
        logging.exception("Test cmd failed, Exception: %s", e.args)


if __name__ == '__main__':
    parse_command_line()

    sql_cmd_1 = 'select * from xxxxxxx'   #  类 xxxxxxx 不存在，非法的sql 语句
    for i in range(3):
        test_cmd(sql_cmd_1)

    sql_cmd_2 = 'select * from V'  #  有效的sql 语句 
    test_cmd(sql_cmd_2)

执行 test.py 文件, 发现：
三次执行 sql_cmd_1 都出现了异常，提示 MemoryError.
之后执行 sql_cmd_2 也出现了同样的异常，提示 MemoryError.

(py2.7.13env) [root@test-app-1 misc]# python test.py
[I 180322 15:56:20 test:19] come into test_cmd
[E 180322 15:56:20 test:30] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 22, in test_cmd
        ret = client.command(sql_cmd)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 462, in command
        .prepare(( QUERY_CMD, ) + args).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/commands.py", line 144, in fetch_response
        super( CommandMessage, self ).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 249, in _decode_all
        self._decode_header()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 176, in _decode_header
        serialized_exception = self._decode_field( FIELD_STRING )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError

[I 180322 15:56:20 test:19] come into test_cmd
[E 180322 15:56:20 test:30] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 21, in test_cmd
        client.db_open(db_name, db_username, db_password)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 412, in db_open
        .prepare((db_name, user, password, db_type, client_id)).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/database.py", line 101, in fetch_response
        result = super(DbOpenMessage, self).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 250, in _decode_all
        self._decode_body()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 242, in _decode_body
        self._body.append( self._decode_field( field ) )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError
[I 180322 15:56:20 test:19] come into test_cmd
[E 180322 15:56:20 test:30] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 21, in test_cmd
        client.db_open(db_name, db_username, db_password)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 412, in db_open
        .prepare((db_name, user, password, db_type, client_id)).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/database.py", line 101, in fetch_response
        result = super(DbOpenMessage, self).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 250, in _decode_all
        self._decode_body()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 242, in _decode_body
        self._body.append( self._decode_field( field ) )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError

[I 180322 15:56:20 test:19] come into test_cmd
[E 180322 15:56:20 test:30] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 21, in test_cmd
        client.db_open(db_name, db_username, db_password)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 412, in db_open
        .prepare((db_name, user, password, db_type, client_id)).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/database.py", line 101, in fetch_response
        result = super(DbOpenMessage, self).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 250, in _decode_all
        self._decode_body()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 242, in _decode_body
        self._body.append( self._decode_field( field ) )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError

查看 OrientDB Server 的日志，可以看到 OQueryParsingException。
虽然OrientDB Server 出现错误日志，但不影响 OrientDB Server 的正常使用，OrientDB Studio 也可以正常访问和使用。

xception `1320FF76` in storage `plocal:/opt/orientdb/databases/cmdb_test`: 2.2.31 (build 285537d2767275f460df32c6a3be01bfff6a517c,
branch 2.2.x)
com.orientechnologies.orient.core.exception.OQueryParsingException: Error on parsing query
Query:  xxxxxxx
------^
        DB name="cmdb_test"
        at com.orientechnologies.orient.core.sql.filter.OSQLTarget.<init>(OSQLTarget.java:74)
        at com.orientechnologies.orient.core.sql.OSQLEngine.parseTarget(OSQLEngine.java:464)
        at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.parse(OCommandExecutorSQLSelect.java:278)
        at com.orientechnologies.orient.core.sql.OCommandExecutorSQLSelect.parse(OCommandExecutorSQLSelect.java:93)
        at com.orientechnologies.orient.core.sql.OCommandExecutorSQLDelegate.parse(OCommandExecutorSQLDelegate.java:53)
        at com.orientechnologies.orient.core.sql.OCommandExecutorSQLDelegate.parse(OCommandExecutorSQLDelegate.java:34)
        at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.command(OAbstractPaginatedStorage.java:331
9)
        at com.orientechnologies.orient.core.command.OCommandRequestTextAbstract.execute(OCommandRequestTextAbstract.java:69)
        at com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary.command(ONetworkProtocolBinary.java:15
64)
        at com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary.executeRequest(ONetworkProtocolBinary.
java:664)
        at com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary.sessionRequest(ONetworkProtocolBinary.
java:398)
        at com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary.execute(ONetworkProtocolBinary.java:21
7)
        at com.orientechnologies.common.thread.OSoftThread.run(OSoftThread.java:82)
Caused by: com.orientechnologies.orient.core.exception.OCommandExecutionException: Class 'XXXXXXX' was not found in database 'cmdb_t
est'
        DB name="cmdb_test"
        at com.orientechnologies.orient.core.sql.filter.OSQLTarget.extractTargets(OSQLTarget.java:273)
        at com.orientechnologies.orient.core.sql.filter.OSQLTarget.<init>(OSQLTarget.java:64)
        ... 12 more

结合日志，分析发现：一旦client.command() 执行出现异常，当前Client 就不能正常使用，即使去执行正常有效的sql_command 也不能执行获取结果集。

一个改进的策略是：

在发现client 不可用的时候（比如 catch 到 MemoryError），重置一下client。
test_cmd方法的修改如下：

def test_cmd(sql_cmd):
    logging.info('come into test_cmd')
    try:
        client.db_open(db_name, db_username, db_password)
        ret = client.command(sql_cmd)
        logging.info('ret len: %s', len(ret))
    except MemoryError as e:
        global client
        client = pyorient.OrientDB(cmdb_url, cmdb_port)
        logging.exception("Test cmd failed, Exception: %s", e.args)
    except Exception as e:
        logging.exception("Test cmd failed, Exception: %s", e.args)

再次执行 test.py, 发现在执行 sql_cmd_1 时出现了 MemoryError。但在后续执行sql_cmd_2 的时候可以正常执行。

(py2.7.13cmdb2.0) [root@test-app-1 misc]# python test.py
test.py:25: SyntaxWarning: name 'client' is used prior to global declaration
  global client
[I 180322 16:13:21 test:19] come into test_cmd
[E 180322 16:13:21 test:27] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 22, in test_cmd
        ret = client.command(sql_cmd)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 462, in command
        .prepare(( QUERY_CMD, ) + args).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/commands.py", line 144, in fetch_response
        super( CommandMessage, self ).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 249, in _decode_all
        self._decode_header()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 176, in _decode_header
        serialized_exception = self._decode_field( FIELD_STRING )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError
[I 180322 16:13:21 test:19] come into test_cmd
[E 180322 16:13:21 test:27] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 22, in test_cmd
        ret = client.command(sql_cmd)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 462, in command
        .prepare(( QUERY_CMD, ) + args).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/commands.py", line 144, in fetch_response
        super( CommandMessage, self ).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 249, in _decode_all
        self._decode_header()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 176, in _decode_header
        serialized_exception = self._decode_field( FIELD_STRING )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError
[I 180322 16:13:21 test:19] come into test_cmd
[E 180322 16:13:21 test:27] Test cmd failed, Exception: ()
    Traceback (most recent call last):
      File "test.py", line 22, in test_cmd
        ret = client.command(sql_cmd)
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 462, in command
        .prepare(( QUERY_CMD, ) + args).send().fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/commands.py", line 144, in fetch_response
        super( CommandMessage, self ).fetch_response()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
        self._decode_all()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 249, in _decode_all
        self._decode_header()
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 176, in _decode_header
        serialized_exception = self._decode_field( FIELD_STRING )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
        _decoded_string = self._orientSocket.read( _len )
      File "/root/.virtualenvs/py2.7.13cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 164, in read
        buf = bytearray(_len_to_read)
    MemoryError
[I 180322 16:13:21 test:19] come into test_cmd
[I 180322 16:13:23 test:23] ret len: 6109

扩展

同样的问题，在交互式命令行，会提示 timeout，这个比较奇怪 :(
执行到 pyorient/orient.py read() 时候，文件执行和命令行执行出现差别...

In [36]: sql_cmd_1 = 'select * from xxxxxxx'

In [37]: test_cmd(sql_cmd_1)
ERROR:root:Test cmd failed, Exception: ('timed out',)
Traceback (most recent call last):
  File "<ipython-input-35-22cb912a84d8>", line 5, in test_cmd
    ret = client.command(sql_cmd)
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 462, in command
    .prepare(( QUERY_CMD, ) + args).send().fetch_response()
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/messages/commands.py", line 144, in fetch_response
    super( CommandMessage, self ).fetch_response()
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 265, in fetch_response
    self._decode_all()
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 249, in _decode_all
    self._decode_header()
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 176, in _decode_header
    serialized_exception = self._decode_field( FIELD_STRING )
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/messages/base.py", line 366, in _decode_field
    _decoded_string = self._orientSocket.read( _len )
  File "/root/.virtualenvs/py2.7.14cmdb2.0/lib/python2.7/site-packages/pyorient/orient.py", line 167, in read
    n_bytes = self._socket.recv_into(view, _len_to_read)
timeout: timed out

附录：

class Exception(BaseException)
 |  Common base class for all non-exit exceptions.
 |
 |  Method resolution order:
 |      Exception
 |      BaseException
 |      __builtin__.object

class MemoryError(StandardError)
 |  Out of memory.
 |
 |  Method resolution order:
 |      MemoryError
 |      StandardError
 |      Exception
 |      BaseException
 |      __builtin__.object
 |

class OrientDB(__builtin__.object)
 |  OrientDB client object
 |
 |  Point of entrance to use the basic commands you can issue to the server
 |
 |  :param host: hostname of the server to connect  defaults to localhost
 |  :param port: integer port of the server         defaults to 2424
 |
 |  Usage::
 |
 |      >>> from pyorient import OrientDB
 |      >>> client = OrientDB("localhost", 2424)
 |      >>> client.db_open('MyDatabase', 'admin', 'admin')
 |
 |  Methods defined here:
 |
 |  __getattr__(self, item)
 |
 |  __init__(self, host='localhost', port=2424, serialization_type='ORecordDocument2csv')
 |
 |  batch(self, *args)
 |
 |  close(self)
 |
 |  command(self, *args)
 |


 |  db_open(self, db_name, user, password, db_type='document', client_id='')
 |       Opens a database on the remote OrientDB Server.
 |       Returns the Session-Id to being reused for all the next calls and the list of configured clusters
 |
 |      :param db_name: database name as string. Example: "demo"
 |      :param user: username as string
 |      :param password: password as string
 |      :param db_type: string, can be DB_TYPE_DOCUMENT or DB_TYPE_GRAPH
 |      :param client_id: Can be null for clients. In clustered configuration is the distributed node
 |      :return: an array of :class:`OrientCluster <pyorient.types.OrientCluster>` object
 |
 |      Usage::
 |
 |        >>> import pyorient
 |        >>> orient = pyorient.OrientDB('localhost', 2424)
 |        >>> orient.db_open('asd', 'admin', 'admin')

OrientDB Server 阻塞问题分析(一)

扩展

附录：

相关文章

网友评论

延伸阅读

深度阅读

栏目导航

热点阅读