diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1077b5f5..41b8e644 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -41,8 +41,6 @@ jobs:
           - "3.4"
         gemfile:
           - gemfiles/standalone.gemfile
-          - gemfiles/rbnacl.gemfile
-          - gemfiles/rbnacl_pre_6.gemfile
         experimental: [false]
         include:
           - os: ubuntu-20.04
@@ -64,11 +62,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install libsodium
-        run: |
-          sudo apt-get update -q
-          sudo apt-get install libsodium-dev -y
-
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -96,7 +89,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Download coverage reports from the test job
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage-reports
 
diff --git a/Appraisals b/Appraisals
index 118db4d4..ca5f125d 100644
--- a/Appraisals
+++ b/Appraisals
@@ -8,13 +8,3 @@ appraise 'openssl' do
   gem 'openssl', '~> 2.1'
   remove_gem 'rubocop'
 end
-
-appraise 'rbnacl' do
-  gem 'rbnacl', '>= 6'
-  remove_gem 'rubocop'
-end
-
-appraise 'rbnacl_pre_6' do
-  gem 'rbnacl', '< 6'
-  remove_gem 'rubocop'
-end
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 94da8216..1b7fe730 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@
 **Breaking changes:**
 - Require token signature to be verified before accessing payload [#648](https://github.com/jwt/ruby-jwt/pull/648) ([@anakinj](https://github.com/anakinj))
 - Drop support for the HS512256 algorithm [#650](https://github.com/jwt/ruby-jwt/pull/650) ([@anakinj](https://github.com/anakinj))
+- Remove dependency to rbnacl [#655](https://github.com/jwt/ruby-jwt/pull/655) ([@anakinj](https://github.com/anakinj))
 
 Take a look at the [upgrade guide](UPGRADING.md) for more details.
 
diff --git a/gemfiles/rbnacl.gemfile b/gemfiles/rbnacl.gemfile
deleted file mode 100644
index ce73f33d..00000000
--- a/gemfiles/rbnacl.gemfile
+++ /dev/null
@@ -1,7 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "rbnacl", ">= 6"
-
-gemspec path: "../"
diff --git a/gemfiles/rbnacl_pre_6.gemfile b/gemfiles/rbnacl_pre_6.gemfile
deleted file mode 100644
index 38c842cf..00000000
--- a/gemfiles/rbnacl_pre_6.gemfile
+++ /dev/null
@@ -1,7 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "rbnacl", "< 6"
-
-gemspec path: "../"
diff --git a/lib/jwt/jwa.rb b/lib/jwt/jwa.rb
index f1b23a4c..8c416ce0 100644
--- a/lib/jwt/jwa.rb
+++ b/lib/jwt/jwa.rb
@@ -2,12 +2,6 @@
 
 require 'openssl'
 
-begin
-  require 'rbnacl'
-rescue LoadError
-  raise if defined?(RbNaCl)
-end
-
 require_relative 'jwa/compat'
 require_relative 'jwa/signing_algorithm'
 require_relative 'jwa/ecdsa'
@@ -18,8 +12,6 @@
 require_relative 'jwa/unsupported'
 require_relative 'jwa/wrapper'
 
-require_relative 'jwa/eddsa' if JWT.rbnacl?
-
 module JWT
   # The JWA module contains all supported algorithms.
   module JWA
diff --git a/lib/jwt/jwa/eddsa.rb b/lib/jwt/jwa/eddsa.rb
deleted file mode 100644
index 441286c8..00000000
--- a/lib/jwt/jwa/eddsa.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module JWT
-  module JWA
-    # Implementation of the EdDSA family of algorithms
-    class Eddsa
-      include JWT::JWA::SigningAlgorithm
-
-      def initialize(alg)
-        @alg = alg
-      end
-
-      def sign(data:, signing_key:)
-        raise_sign_error!("Key given is a #{signing_key.class} but has to be an RbNaCl::Signatures::Ed25519::SigningKey") unless signing_key.is_a?(RbNaCl::Signatures::Ed25519::SigningKey)
-
-        Deprecations.warning('Using the EdDSA algorithm is deprecated and will be removed in a future version of ruby-jwt. In the future the algorithm will be provided by the jwt-eddsa gem.')
-
-        signing_key.sign(data)
-      end
-
-      def verify(data:, signature:, verification_key:)
-        raise_verify_error!("key given is a #{verification_key.class} but has to be a RbNaCl::Signatures::Ed25519::VerifyKey") unless verification_key.is_a?(RbNaCl::Signatures::Ed25519::VerifyKey)
-
-        Deprecations.warning('Using the EdDSA algorithm is deprecated and will be removed in a future version of ruby-jwt. In the future the algorithm will be provided by the jwt-eddsa gem.')
-
-        verification_key.verify(signature, data)
-      rescue RbNaCl::CryptoError
-        false
-      end
-
-      register_algorithm(new('ED25519'))
-      register_algorithm(new('EdDSA'))
-    end
-  end
-end
diff --git a/lib/jwt/jwk.rb b/lib/jwt/jwk.rb
index 216351ea..d717bac7 100644
--- a/lib/jwt/jwk.rb
+++ b/lib/jwt/jwk.rb
@@ -53,4 +53,3 @@ def generate_mappings
 require_relative 'jwk/ec'
 require_relative 'jwk/rsa'
 require_relative 'jwk/hmac'
-require_relative 'jwk/okp_rbnacl' if JWT.rbnacl?
diff --git a/lib/jwt/jwk/okp_rbnacl.rb b/lib/jwt/jwk/okp_rbnacl.rb
deleted file mode 100644
index d0e72492..00000000
--- a/lib/jwt/jwk/okp_rbnacl.rb
+++ /dev/null
@@ -1,109 +0,0 @@
-# frozen_string_literal: true
-
-module JWT
-  module JWK
-    # JSON Web Key (JWK) representation for Ed25519 keys
-    class OKPRbNaCl < KeyBase
-      KTY  = 'OKP'
-      KTYS = [KTY, JWT::JWK::OKPRbNaCl, RbNaCl::Signatures::Ed25519::SigningKey, RbNaCl::Signatures::Ed25519::VerifyKey].freeze
-      OKP_PUBLIC_KEY_ELEMENTS = %i[kty n x].freeze
-      OKP_PRIVATE_KEY_ELEMENTS = %i[d].freeze
-
-      def initialize(key, params = nil, options = {})
-        params ||= {}
-        Deprecations.warning('Using the OKP JWK for Ed25519 keys is deprecated and will be removed in a future version of ruby-jwt. Please use the ruby-eddsa gem instead.')
-        # For backwards compatibility when kid was a String
-        params = { kid: params } if params.is_a?(String)
-
-        key_params = extract_key_params(key)
-
-        params = params.transform_keys(&:to_sym)
-        check_jwk_params!(key_params, params)
-        super(options, key_params.merge(params))
-      end
-
-      def verify_key
-        return @verify_key if defined?(@verify_key)
-
-        @verify_key = verify_key_from_parameters
-      end
-
-      def signing_key
-        return @signing_key if defined?(@signing_key)
-
-        @signing_key = signing_key_from_parameters
-      end
-
-      def key_digest
-        Thumbprint.new(self).to_s
-      end
-
-      def private?
-        !signing_key.nil?
-      end
-
-      def members
-        OKP_PUBLIC_KEY_ELEMENTS.each_with_object({}) { |i, h| h[i] = self[i] }
-      end
-
-      def export(options = {})
-        exported = parameters.clone
-        exported.reject! { |k, _| OKP_PRIVATE_KEY_ELEMENTS.include?(k) } unless private? && options[:include_private] == true
-        exported
-      end
-
-      private
-
-      def extract_key_params(key)
-        case key
-        when JWT::JWK::KeyBase
-          key.export(include_private: true)
-        when RbNaCl::Signatures::Ed25519::SigningKey
-          @signing_key = key
-          @verify_key = key.verify_key
-          parse_okp_key_params(@verify_key, @signing_key)
-        when RbNaCl::Signatures::Ed25519::VerifyKey
-          @signing_key = nil
-          @verify_key = key
-          parse_okp_key_params(@verify_key)
-        when Hash
-          key.transform_keys(&:to_sym)
-        else
-          raise ArgumentError, 'key must be of type RbNaCl::Signatures::Ed25519::SigningKey, RbNaCl::Signatures::Ed25519::VerifyKey or Hash with key parameters'
-        end
-      end
-
-      def check_jwk_params!(key_params, _given_params)
-        raise JWT::JWKError, "Incorrect 'kty' value: #{key_params[:kty]}, expected #{KTY}" unless key_params[:kty] == KTY
-      end
-
-      def parse_okp_key_params(verify_key, signing_key = nil)
-        params = {
-          kty: KTY,
-          crv: 'Ed25519',
-          x: ::JWT::Base64.url_encode(verify_key.to_bytes)
-        }
-
-        params[:d] = ::JWT::Base64.url_encode(signing_key.to_bytes) if signing_key
-
-        params
-      end
-
-      def verify_key_from_parameters
-        RbNaCl::Signatures::Ed25519::VerifyKey.new(::JWT::Base64.url_decode(self[:x]))
-      end
-
-      def signing_key_from_parameters
-        return nil unless self[:d]
-
-        RbNaCl::Signatures::Ed25519::SigningKey.new(::JWT::Base64.url_decode(self[:d]))
-      end
-
-      class << self
-        def import(jwk_data)
-          new(jwk_data)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/jwt/version.rb b/lib/jwt/version.rb
index 37309a97..ced5932a 100644
--- a/lib/jwt/version.rb
+++ b/lib/jwt/version.rb
@@ -32,22 +32,6 @@ def self.openssl_3?
     true if 3 * 0x10000000 <= OpenSSL::OPENSSL_VERSION_NUMBER
   end
 
-  # Checks if the RbNaCl library is defined.
-  #
-  # @return [Boolean] true if RbNaCl is defined, false otherwise.
-  # @api private
-  def self.rbnacl?
-    defined?(::RbNaCl)
-  end
-
-  # Checks if the RbNaCl library version is 6.0.0 or greater.
-  #
-  # @return [Boolean] true if RbNaCl version is 6.0.0 or greater, false otherwise.
-  # @api private
-  def self.rbnacl_6_or_greater?
-    rbnacl? && ::Gem::Version.new(::RbNaCl::VERSION) >= ::Gem::Version.new('6.0.0')
-  end
-
   # Checks if there is an OpenSSL 3 HMAC empty key regression.
   #
   # @return [Boolean] true if there is an OpenSSL 3 HMAC empty key regression, false otherwise.
diff --git a/spec/integration/readme_examples_spec.rb b/spec/integration/readme_examples_spec.rb
index 9f84058f..3c8ecc80 100644
--- a/spec/integration/readme_examples_spec.rb
+++ b/spec/integration/readme_examples_spec.rb
@@ -65,21 +65,6 @@
       ]
     end
 
-    if defined?(RbNaCl)
-      it 'EDDSA' do
-        eddsa_key = RbNaCl::Signatures::Ed25519::SigningKey.generate
-        eddsa_public = eddsa_key.verify_key
-
-        token = JWT.encode payload, eddsa_key, 'ED25519'
-        decoded_token = JWT.decode token, eddsa_public, true, algorithm: 'ED25519'
-
-        expect(decoded_token).to eq [
-          { 'data' => 'test' },
-          { 'alg' => 'ED25519' }
-        ]
-      end
-    end
-
     if Gem::Version.new(OpenSSL::VERSION) >= Gem::Version.new('2.1')
       it 'RSASSA-PSS' do
         rsa_private = OpenSSL::PKey::RSA.generate 2048
diff --git a/spec/jwt/jwa/eddsa_spec.rb b/spec/jwt/jwa/eddsa_spec.rb
deleted file mode 100644
index eae97509..00000000
--- a/spec/jwt/jwa/eddsa_spec.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe 'JWT::JWA::Eddsa' do
-  let(:key) { RbNaCl::Signatures::Ed25519::SigningKey.generate }
-
-  before do
-    skip('Requires the rbnacl gem') unless JWT.rbnacl?
-  end
-
-  context 'backwards compatibility' do
-    it 'signs and verifies' do
-      signature = JWT::JWA::Eddsa.sign('RS256', 'data', key)
-      expect(JWT::JWA::Eddsa.verify('RS256', key.verify_key, 'data', signature)).to be(true)
-    end
-  end
-
-  context 'when when signing with invalid RbNaCl::Signatures::Ed25519::SigningKey' do
-    it 'raises an error' do
-      expect do
-        JWT::JWA::Eddsa.sign('RS256', 'data', 'key')
-      end.to raise_error(JWT::EncodeError, 'Key given is a String but has to be an RbNaCl::Signatures::Ed25519::SigningKey')
-    end
-  end
-
-  context 'when when verifying with invalid RbNaCl::Signatures::Ed25519::VerifyKey' do
-    it 'raises an error' do
-      expect do
-        JWT::JWA::Eddsa.verify('RS256', 'key', 'data', 'signature')
-      end.to raise_error(JWT::DecodeError, 'key given is a String but has to be a RbNaCl::Signatures::Ed25519::VerifyKey')
-    end
-  end
-end
diff --git a/spec/jwt/jwk/decode_with_jwk_spec.rb b/spec/jwt/jwk/decode_with_jwk_spec.rb
index d55ff64b..62d82638 100644
--- a/spec/jwt/jwk/decode_with_jwk_spec.rb
+++ b/spec/jwt/jwk/decode_with_jwk_spec.rb
@@ -162,23 +162,6 @@
           )
         end
       end
-
-      if defined?(RbNaCl)
-        context 'when OKP keys are used' do
-          before do
-            skip('Requires the rbnacl gem') unless JWT.rbnacl?
-          end
-
-          let(:keypair) { RbNaCl::Signatures::Ed25519::SigningKey.new(SecureRandom.hex) }
-          let(:algorithm) { 'ED25519' }
-
-          it 'decodes the token' do
-            key_loader = ->(_options) { JSON.parse(JSON.generate(public_jwks)) }
-            payload, _header = described_class.decode(signed_token, nil, true, { algorithms: [algorithm], jwks: key_loader })
-            expect(payload).to eq(token_payload)
-          end
-        end
-      end
     end
   end
 end
diff --git a/spec/jwt/jwk/okp_rbnacl_spec.rb b/spec/jwt/jwk/okp_rbnacl_spec.rb
deleted file mode 100644
index 88303163..00000000
--- a/spec/jwt/jwk/okp_rbnacl_spec.rb
+++ /dev/null
@@ -1,128 +0,0 @@
-# frozen_string_literal: true
-
-require 'securerandom'
-
-RSpec.describe 'JWT::JWK::OKPRbNaCl' do
-  let(:described_class) { JWT::JWK::OKPRbNaCl }
-  let(:private_key) { RbNaCl::Signatures::Ed25519::SigningKey.new(SecureRandom.hex) }
-  let(:public_key)  { private_key.verify_key }
-  let(:key)         { nil }
-
-  subject(:instance) { described_class.new(key) }
-
-  before do
-    skip('Requires the rbnacl gem') unless JWT.rbnacl?
-  end
-
-  describe '.new' do
-    context 'when private key is given' do
-      let(:key) { private_key }
-      it { is_expected.to be_a(described_class) }
-    end
-    context 'when public key is given' do
-      let(:key) { public_key }
-      it { is_expected.to be_a(described_class) }
-    end
-    context 'when something else than a public or private key is given' do
-      let(:key) { OpenSSL::PKey::RSA.new(2048) }
-      it 'raises an ArgumentError' do
-        expect { instance }.to raise_error(ArgumentError)
-      end
-    end
-
-    context 'when jwk parameters given' do
-      let(:key) do
-        {
-          kty: 'OKP',
-          use: 'sig',
-          crv: 'Ed25519',
-          kid: '27zV',
-          x: '0I6olrZGYml7JGusuKJW9G7D0DZ9UormSady9kR7V4Q'
-        }
-      end
-      it { is_expected.to be_a(described_class) }
-    end
-  end
-
-  describe '#verify_key' do
-    let(:key) { private_key }
-    subject { instance.verify_key }
-    it 'is the verify key' do
-      expect(subject).to be_a(RbNaCl::Signatures::Ed25519::VerifyKey)
-    end
-  end
-
-  describe '#private?' do
-    subject { instance.private? }
-    context 'when private key is given' do
-      let(:key) { private_key }
-      it { is_expected.to eq(true) }
-    end
-    context 'when public key is given' do
-      let(:key) { public_key }
-      it { is_expected.to eq(false) }
-    end
-  end
-
-  describe '#export' do
-    let(:options) { {} }
-    subject { instance.export(options) }
-    context 'when private key is given' do
-      let(:key) { private_key }
-      it 'exports the public key' do
-        expect(subject).to include(crv: 'Ed25519', kty: 'OKP')
-        expect(subject.keys).to eq(%i[kty crv x kid])
-        expect(subject[:x].size).to eq(43)
-        expect(subject[:kid].size).to eq(43)
-      end
-    end
-    context 'when private key is asked for' do
-      let(:key) { private_key }
-      let(:options) { { include_private: true } }
-      it 'exports the private key' do
-        expect(subject).to include(crv: 'Ed25519', kty: 'OKP')
-        expect(subject.keys).to eq(%i[kty crv x d kid])
-        expect(subject[:x].size).to eq(43)
-        expect(subject[:d].size).to eq(43)
-        expect(subject[:kid].size).to eq(43)
-      end
-    end
-  end
-
-  describe '.import' do
-    subject { described_class.import(import_data) }
-
-    context 'when exported public key is given' do
-      let(:import_data) { described_class.new(public_key).export }
-      it 'creates a new instance of the class' do
-        expect(subject.private?).to eq(false)
-        expect(subject.verify_key).to be_a(RbNaCl::Signatures::Ed25519::VerifyKey)
-        expect(subject.signing_key).to eq(nil)
-        expect(subject.verify_key.to_bytes).to eq(public_key.to_bytes)
-        expect(subject.kid).to eq(import_data[:kid])
-      end
-    end
-
-    context 'when exported private key is given' do
-      let(:import_data) { described_class.new(private_key).export(include_private: true) }
-      it 'creates a new instance of the class' do
-        expect(subject.private?).to eq(true)
-        expect(subject.verify_key).to be_a(RbNaCl::Signatures::Ed25519::VerifyKey)
-        expect(subject.signing_key).to be_a(RbNaCl::Signatures::Ed25519::SigningKey)
-        expect(subject.verify_key.to_bytes).to eq(public_key.to_bytes)
-        expect(subject.kid).to eq(import_data[:kid])
-      end
-    end
-
-    context 'when JWK is given' do
-      let(:import_data) { described_class.new(private_key) }
-      it 'creates a new instance of the class' do
-        expect(subject.private?).to eq(true)
-        expect(subject.verify_key).to be_a(RbNaCl::Signatures::Ed25519::VerifyKey)
-        expect(subject.signing_key).to be_a(RbNaCl::Signatures::Ed25519::SigningKey)
-        expect(subject.verify_key.to_bytes).to eq(public_key.to_bytes)
-        expect(subject.kid).to eq(import_data[:kid])
-      end
-    end
-  end
-end
diff --git a/spec/jwt/jwt_spec.rb b/spec/jwt/jwt_spec.rb
index 1e67307c..7488e438 100644
--- a/spec/jwt/jwt_spec.rb
+++ b/spec/jwt/jwt_spec.rb
@@ -3,8 +3,8 @@
 RSpec.describe JWT do
   let(:payload) { { 'user_id' => 'some@user.tld' } }
 
-  let :data do
-    data = {
+  let(:data) do
+    {
       :empty_token => 'e30K.e30K.e30K',
       :empty_token_2_segment => 'e30K.e30K.',
       :invalid_header_token => 'W10.e30K.e30K',
@@ -37,18 +37,6 @@
       'PS384' => '',
       'PS512' => ''
     }
-
-    if JWT.rbnacl?
-      ed25519_private = RbNaCl::Signatures::Ed25519::SigningKey.new('abcdefghijklmnopqrstuvwxyzABCDEF')
-      ed25519_public =  ed25519_private.verify_key
-      data.merge!(
-        'ED25519_private' => ed25519_private,
-        'ED25519_public' => ed25519_public,
-        'EdDSA_private' => ed25519_private,
-        'EdDSA_public' => ed25519_public
-      )
-    end
-    data
   end
 
   after(:each) do
@@ -174,44 +162,6 @@
     end
   end
 
-  if defined?(RbNaCl)
-    %w[ED25519 EdDSA].each do |alg|
-      context "alg: #{alg}" do
-        before(:each) do
-          data[alg] = JWT.encode payload, data["#{alg}_private"], alg
-        end
-
-        let(:wrong_key) { RbNaCl::Signatures::Ed25519::SigningKey.generate.verify_key }
-
-        it 'should generate a valid token' do
-          jwt_payload, header = JWT.decode data[alg], data["#{alg}_public"], true, algorithm: alg
-
-          expect(header['alg']).to eq alg
-          expect(jwt_payload).to eq payload
-        end
-
-        it 'should decode a valid token' do
-          jwt_payload, header = JWT.decode data[alg], data["#{alg}_public"], true, algorithm: alg
-
-          expect(header['alg']).to eq alg
-          expect(jwt_payload).to eq payload
-        end
-
-        it 'wrong key should raise JWT::DecodeError' do
-          expect do
-            JWT.decode data[alg], wrong_key, true, algorithm: alg
-          end.to raise_error JWT::DecodeError
-        end
-
-        it 'wrong key and verify = false should not raise JWT::DecodeError' do
-          expect do
-            JWT.decode data[alg], wrong_key, false
-          end.not_to raise_error
-        end
-      end
-    end
-  end
-
   %w[ES256 ES384 ES512 ES256K].each do |alg|
     context "alg: #{alg}" do
       before(:each) do