diff --git a/src/cowboy_http2.erl b/src/cowboy_http2.erl
index f4203998..0e110cd9 100644
--- a/src/cowboy_http2.erl
+++ b/src/cowboy_http2.erl
@@ -321,8 +321,13 @@ set_idle_timeout(State=#state{http2_status=Status, timer=TimerRef}, _)
 			TimerRef =/= undefined ->
 	State;
 set_idle_timeout(State=#state{opts=Opts}, TimeoutNum) ->
-	set_timeout(State#state{idle_timeout_num=TimeoutNum},
-		maps:get(idle_timeout, Opts, 60000), idle_timeout).
+	case maps:get(idle_timeout, Opts, 60000) of
+		infinity ->
+			State#state{timer=undefined};
+		Timeout ->
+			set_timeout(State#state{idle_timeout_num=TimeoutNum},
+				Timeout div ?IDLE_TIMEOUT_TICKS, idle_timeout)
+	end.
 
 set_timeout(State=#state{timer=TimerRef0}, Timeout, Message) ->
 	ok = case TimerRef0 of