[aarch64, debug] topology_custom/test_tablets2 failed with PytestCollectionWarning:

[scylladb-promoter]

https://jenkins.scylladb.com/job/scylla-master/job/next/8649/ failed with the following error:

_______________ test_tombstone_gc_disabled_on_pending_replica.1 ________________

manager = <test.pylib.manager_client.ManagerClient object at 0xffff683e0e90>

    @pytest.mark.asyncio
    @skip_mode('release', 'error injections are not supported in release mode')
    async def test_tombstone_gc_disabled_on_pending_replica(manager: ManagerClient):
        logger.info("Bootstrapping cluster")
        servers = [await manager.server_add()]
    
        await manager.api.disable_tablet_balancing(servers[0].ip_addr)
    
        cql = manager.get_cql()
        await cql.run_async("CREATE KEYSPACE test WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': 1} AND tablets = {'initial': 4};")
        await cql.run_async("CREATE TABLE test.test (pk int PRIMARY KEY, c int) WITH gc_grace_seconds = 0;")
    
        servers.append(await manager.server_add())
    
        key = 7 # Whatever
        tablet_token = 0 # Doesn't matter since there is one tablet
        await cql.run_async(f"INSERT INTO test.test (pk, c) VALUES ({key}, 1) USING timestamp 9")
        rows = await cql.run_async("SELECT pk from test.test")
        assert len(rows) == 1
    
        replica = await get_tablet_replica(manager, servers[0], 'test', 'test', tablet_token)
    
        s0_host_id = await manager.get_host_id(servers[0].server_id)
        s1_host_id = await manager.get_host_id(servers[1].server_id)
        dst_shard = 0
    
        await manager.api.enable_injection(servers[1].ip_addr, "stream_mutation_fragments", one_shot=True)
        s1_log = await manager.server_open_log(servers[1].server_id)
        s1_mark = await s1_log.mark()
    
        migration_task = asyncio.create_task(
            manager.api.move_tablet(servers[0].ip_addr, "test", "test", replica[0], replica[1], s1_host_id, dst_shard, tablet_token))
    
        await s1_log.wait_for('stream_mutation_fragments: waiting', from_mark=s1_mark)
        s1_mark = await s1_log.mark()
    
        # write a tombstone with timestamp X to DB
        await cql.run_async(f'DELETE FROM test.test USING timestamp 10 WHERE pk = {key}')
    
        # flush both servers
        for s in servers:
            await manager.api.flush_keyspace(s.ip_addr, "test")
    
        await asyncio.sleep(1)
    
        # major compact both servers
        for s in servers:
            await manager.api.keyspace_compaction(s.ip_addr, "test")
    
        # write backdated data to test.test with timestamp X-1 with the same key as the tombstone
        await cql.run_async(f'INSERT INTO test.test (pk, c) VALUES ({key}, 0) USING timestamp 9')
    
        # release streaming
        await manager.api.message_injection(servers[1].ip_addr, "stream_mutation_fragments")
        await s1_log.wait_for('stream_mutation_fragments: done', from_mark=s1_mark)
    
        logger.info("Waiting for migration to finish")
        await migration_task
        logger.info("Migration done")
    
        for s in servers:
            await manager.api.flush_keyspace(s.ip_addr, "test")
    
        # verify result
        rows = await cql.run_async(f'SELECT pk, c FROM test.test WHERE pk = {key};')
>       assert len(rows) == 0
E       assert 1 == 0
E        +  where 1 = len([Row(pk=7, c=0)])

test/topology_custom/test_tablets2.py:1388: AssertionError
------------------------------ Captured log setup ------------------------------

[mykaul]

pytest (2).log
test_py (1).log
scylla-856.log
scylla-855.log

[mykaul]

@bhalevy - please assign someone to look at (I did not see this issue before)

[bhalevy]

@nodep I see you added this test in cdf775d
I wonder if it's valid... can't tombstone_gc happen right after tablet migration completes?

[nodep]

@bhalevy Yes, you are right tombstone_gc can happen after the migration, which could explain what happened here. I will look into this to validate the hypothesis.