update_q.py

    def update_q(self, states, actions, rewards, dones, next_states):

        rewards = tf.clip_by_value(tf.reshape(rewards, (-1, 1)), -1.0, 1.0)
        dones = tf.reshape(dones, (-1, 1))

        target_q = rewards + self.gamma * (1.0 - dones) * self.valuenet(next_states)

        with tf.GradientTape() as tape:
            q1, q2 = self.qnet(states, actions)
            loss = tf.reduce_mean(
                tf.square(target_q - q1) + tf.square(target_q - q2)
            )

        variables = self.qnet.trainable_variables
        grads = tape.gradient(loss, variables)
        self.q_optimizer.apply_gradients(zip(grads, variables))

        return loss